1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616 if let Some(startup) = &mut state.startup {
617 startup.stage = stage.to_owned();
618 }
619 state.save()
620}
621
622struct StartupGuard {
625 id: String,
626 armed: bool,
627}
628
629impl StartupGuard {
630 fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631 state.startup = Some(crate::run::Startup {
632 kind: kind.to_owned(),
633 branch: branch.map(str::to_owned),
634 stage: "reserved".to_owned(),
635 complete: false,
636 failure: None,
637 });
638 state.status = RunStatus::Blocked;
639 state.reserve()?;
640 Ok(Self {
641 id: state.id.clone(),
642 armed: true,
643 })
644 }
645
646 fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647 startup_stage(state, stage)
648 }
649
650 fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651 let result = result.and_then(|()| {
652 state.startup.as_mut().expect("reserved startup").complete = true;
653 state.save()
654 });
655 self.armed = false;
656 match result {
657 Ok(()) => Ok(()),
658 Err(error) => {
659 let startup = state.startup.as_mut().expect("reserved startup");
660 startup.complete = false;
661 startup.failure = Some(format!("{error:#}"));
662 let context = crate::run::StartupFailure {
663 run: state.id.clone(),
664 stage: startup.stage.clone(),
665 };
666 state.status = RunStatus::Failed;
669 state.driver_exited = true;
670 state.event(
671 "startup-failed",
672 format!("{context}: {error:#}; retained assets require inspection"),
673 );
674 if let Err(save) = state.save() {
675 tracing::warn!(
676 "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677 state.id
678 );
679 }
680 Err(error.context(context))
681 }
682 }
683 }
684}
685
686impl Drop for StartupGuard {
687 fn drop(&mut self) {
688 if self.armed
689 && let Ok(mut state) = RunState::load(&self.id)
690 {
691 state.status = RunStatus::Blocked;
692 state.driver_exited = true;
693 if let Some(startup) = &mut state.startup {
694 startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695 }
696 state.event(
697 "startup-interrupted",
698 "startup did not finish; inspect retained assets before recovery",
699 );
700 if let Err(error) = state.save() {
701 tracing::error!(
702 "could not record interrupted startup {}: {error:#}",
703 self.id
704 );
705 }
706 }
707 }
708}
709
710impl Runner {
711 pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713 if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714 return error;
715 }
716 if let Some(startup) = &mut self.state.startup {
717 startup.stage = stage.to_owned();
718 } else {
719 return error;
721 }
722 StartupGuard {
723 id: self.state.id.clone(),
724 armed: true,
725 }
726 .finish(&mut self.state, Err(error))
727 .expect_err("startup failed")
728 }
729
730 pub async fn start(
732 repo: &Path,
733 instruction: String,
734 config: Config,
735 origin: Origin,
736 ) -> Result<Self> {
737 Self::start_naming(repo, instruction, "", config, origin).await
738 }
739
740 pub async fn start_naming(
743 repo: &Path,
744 instruction: String,
745 _also_scan: &str,
746 config: Config,
747 origin: Origin,
748 ) -> Result<Self> {
749 let repo = git::toplevel(repo).await?;
750 let missing = agent::missing_programs(&config.agents);
751 if !missing.is_empty() {
752 bail!(
753 "these agent programs are not on PATH: {}. Fix the roster in \
754 magi.toml or install them.",
755 missing.join(", ")
756 );
757 }
758 let base_branch =
759 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760 .await?;
761 let fetched_base = resolve_base(
762 &repo,
763 &base_branch,
764 &config.merge.remote,
765 config.coordination.as_ref(),
766 )
767 .await?;
768 let base_commit = fetched_base.commit.clone();
769 let roles = config.resolve_roles()?;
770 let max_parallel = config.graph.max_parallel.max(1);
771 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772 state.fetched_base = Some(fetched_base);
776 state.origin = Some(origin);
777 let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778 state.status = RunStatus::Prep;
779 state.event("start", format!("run {} created", state.id));
780 startup.finish(&mut state, Ok(()))?;
781 Ok(Self {
782 state,
783 roles,
784 sem: Arc::new(Semaphore::new(max_parallel)),
785 pause: Pause::new(),
786 interrupt: Pause::new(),
787 })
788 }
789
790 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804 Self::review_taking_over(repo, branch, config, None, origin).await
805 }
806
807 pub async fn review_taking_over(
814 repo: &Path,
815 branch: &str,
816 config: Config,
817 takeover: Option<crate::handover::Takeover>,
818 origin: Origin,
819 ) -> Result<Self> {
820 let repo = git::toplevel(repo).await?;
821 let missing = agent::missing_programs(&config.agents);
822 if !missing.is_empty() {
823 bail!(
824 "these agent programs are not on PATH: {}. Fix the roster in \
825 magi.toml or install them.",
826 missing.join(", ")
827 );
828 }
829 let base_branch =
830 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831 .await?;
832 if base_branch == branch {
833 bail!("`{branch}` is the base branch; there is nothing to review against");
834 }
835 let fetched_base = resolve_base(
836 &repo,
837 &base_branch,
838 &config.merge.remote,
839 config.coordination.as_ref(),
840 )
841 .await?;
842 let base_commit = fetched_base.commit.clone();
843
844 let roles = config.resolve_roles()?;
845 let max_parallel = config.graph.max_parallel.max(1);
846 let mut state = RunState::new(
847 repo.clone(),
848 base_branch,
849 base_commit.clone(),
850 format!("Review the work already on branch `{branch}`."),
851 config,
852 );
853 state.fetched_base = Some(fetched_base);
854 state.origin = Some(origin);
855 state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856 let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857 let opened = async {
858 startup.stage(&mut state, "handover")?;
859
860 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867 earlier: Vec::new(),
868 home: crate::run::home(),
869 choice: None,
870 task_instruction: None,
871 });
872 let wait = repository_wait(&state, None);
873 let coordination_config = state.config.coordination.clone();
874 let coordination =
875 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876 .await?;
877 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878 if let Some(released) = &released {
879 state.event(
880 "release",
881 format!(
882 "took `{branch}` over from run {}: its worktree was released: {}",
883 crate::run::short_of(&released.old_id),
884 released.audit
885 ),
886 );
887 }
888 let opened = async {
889 startup.stage(&mut state, "review-preflight")?;
890 state.foreign_checkout = crate::handover::preflight_review(
894 &repo,
895 branch,
896 &takeover.home,
897 &state.config.merge.remote,
898 )
899 .await?;
900 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902 }
903 if let Some(choice) = takeover.choice.as_ref()
904 && let Err(e) =
905 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906 .await
907 {
908 return Err(e.context("applying the owner's answer about the diverged branch"));
909 }
910 for id in takeover.earlier.iter().rev() {
916 if let Ok(old) = RunState::load_under(id, &takeover.home)
917 && let Some(head) = old.pr_head.clone()
918 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919 {
920 state.pr_head = Some(head);
921 break;
922 }
923 }
924 drop(coordination);
925 startup.stage(&mut state, "open-review")?;
926 Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927 }
928 .await;
929 if opened.is_err()
930 && let Some(released) = &released
931 {
932 match crate::repo_lock::SeatLease::operation_with(
935 &repo,
936 coordination_config.as_ref(),
937 &wait,
938 )
939 .await
940 {
941 Ok(_coordination) => {
942 released.restore(&repo, branch).await;
943 let outcome = RunState::load_under(&released.old_id, &takeover.home)
944 .map(|old| {
945 format!(
946 "predecessor released_to={:?}; candidate path exists={}",
947 old.released_to,
948 old.candidates.iter().any(|c| {
949 c.branch == branch && c.worktree.exists()
950 }),
951 )
952 })
953 .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954 state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955 }
956 Err(error) => {
957 tracing::warn!(
958 "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959 state.id
960 );
961 state.event(
962 "startup-restore",
963 format!("restoration not attempted: {error:#}"),
964 );
965 }
966 }
967 }
968 opened
969 }
970 .await;
971 startup.finish(&mut state, opened)?;
972 Ok(Self {
973 state,
974 roles,
975 sem: Arc::new(Semaphore::new(max_parallel)),
976 pause: Pause::new(),
977 interrupt: Pause::new(),
978 })
979 }
980
981 pub async fn recover_pr(
983 repo: &Path,
984 task: &str,
985 selection: crate::pr_recovery::Selection,
986 mut config: Config,
987 ) -> Result<Self> {
988 selection.configure(&mut config)?;
989 selection.check_task(task, &selection.instruction)?;
990 let base_branch =
991 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992 .await?;
993 let fetched_base = resolve_base(
994 repo,
995 &base_branch,
996 &config.merge.remote,
997 config.coordination.as_ref(),
998 )
999 .await?;
1000 let base_commit = fetched_base.commit.clone();
1001 selection
1002 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003 .await?;
1004 Box::pin(selection.verify_source(repo)).await?;
1005 if let Some(proof) = &selection.source_proof {
1006 anyhow::ensure!(
1007 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008 "source rebase base is outside the fetched landing base"
1009 );
1010 }
1011 let roles = config.resolve_roles()?;
1012 let max_parallel = config.graph.max_parallel.max(1);
1013 let mut state = RunState::new(
1014 repo.to_path_buf(),
1015 base_branch,
1016 base_commit.clone(),
1017 selection.instruction.clone(),
1018 config,
1019 );
1020 state.fetched_base = Some(fetched_base);
1021 state.origin = Some(crate::run::Origin::queue(task));
1022 state.pr_head = Some(selection.branch.clone());
1023 state.pr_recovery = Some(selection.clone());
1024 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025 }
1026
1027 async fn open_selected(
1028 repo: &Path,
1029 mut state: RunState,
1030 roles: ResolvedRoles,
1031 max_parallel: usize,
1032 base_commit: String,
1033 selection: crate::pr_recovery::Selection,
1034 ) -> Result<Self> {
1035 let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036 let opened = async {
1037 startup.stage(&mut state, "predecessors")?;
1038 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039 startup.stage(&mut state, "open-selected")?;
1040 let branch = state.branch_for('A');
1041 let wait = repository_wait(&state, None);
1042 let prepared = async {
1043 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044 git::git(repo, &["branch", &branch, selection.source()]).await?;
1045 drop(coordination);
1046 anyhow::Ok(())
1047 }
1048 .await;
1049 prepared?;
1050 state.event(
1051 "recovery",
1052 format!(
1053 "selected implementation {} for {} (public lease {})",
1054 selection.source(),
1055 selection.url,
1056 selection.head
1057 ),
1058 );
1059 let takeover = crate::handover::Takeover {
1060 earlier: Vec::new(),
1061 home: crate::run::home(),
1062 choice: None,
1063 task_instruction: Some(selection.instruction.clone()),
1064 };
1065 state.save()?;
1066 Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067 }
1068 .await;
1069 startup.finish(&mut state, opened)?;
1070 Ok(Self {
1071 state,
1072 roles,
1073 sem: Arc::new(Semaphore::new(max_parallel)),
1074 pause: Pause::new(),
1075 interrupt: Pause::new(),
1076 })
1077 }
1078
1079 async fn inherit(
1084 repo: &Path,
1085 branch: &str,
1086 base_commit: &str,
1087 takeover: &crate::handover::Takeover,
1088 ) -> Option<crate::recovery::RecoveryRecord> {
1089 use crate::recovery::{Relation, assess};
1090 let newest = takeover.earlier.last()?;
1091 let mut chosen = None;
1092 for id in takeover.earlier.iter().rev() {
1093 if let Ok(old) = RunState::load_under(id, &takeover.home)
1094 && !old.reviews.is_empty()
1095 && (old.winner().is_some_and(|w| w.branch == branch)
1096 || old.pr_head.as_deref() == Some(branch))
1097 {
1098 chosen = Some(old);
1099 break;
1100 }
1101 }
1102 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103 .await
1104 .unwrap_or_default();
1105 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106 Some(last) if last.head == tip => Relation::Same,
1107 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108 _ => Relation::Diverged,
1109 };
1110 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111 Some(assess(from, chosen.as_ref(), base_commit, relation))
1112 }
1113
1114 async fn open_review(
1117 repo: &Path,
1118 branch: &str,
1119 state: &mut RunState,
1120 base_commit: String,
1121 takeover: &crate::handover::Takeover,
1122 ) -> Result<()> {
1123 startup_stage(state, "review-sync")?;
1124 let wait = repository_wait(state, None);
1125 let _coordination = coordinate_operation(state, repo, &wait).await?;
1126 if state.foreign_checkout.is_some() {
1127 if state.pr_head.is_none() {
1129 state.pr_head = Some(branch.to_owned());
1130 }
1131 } else if state.pr_recovery.is_none() {
1132 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133 }
1134 let start = review_base(
1137 repo,
1138 &state.config.merge.remote,
1139 &state.base_branch,
1140 &base_commit,
1141 branch,
1142 )
1143 .await;
1144 let log = git::log_oneline(repo, &start, branch)
1145 .await
1146 .unwrap_or_default();
1147 let log_text = if log.trim().is_empty() {
1148 "(no commit messages)"
1149 } else {
1150 log.trim()
1151 };
1152 let mut instruction = format!(
1153 "Review the work already on branch `{branch}`. There is no task \
1154 statement: what the change claims to do is whatever its commits \
1155 say.\n\n{log_text}"
1156 );
1157 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161 if let Some(task) = takeover
1162 .task_instruction
1163 .as_deref()
1164 .filter(|t| !t.trim().is_empty())
1165 && rec.reviewed_head.is_some()
1166 {
1167 instruction = format!(
1168 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169 task.trim()
1170 );
1171 rec.criteria_inherited = true;
1172 }
1173 state.event("recovery", rec.summary());
1174 state.recovery = Some(rec);
1175 }
1176 if let Some(selected) = &state.pr_recovery {
1177 instruction = selected.instruction.clone();
1178 }
1179 state.task_instruction = takeover.task_instruction.clone();
1180 state.instruction = instruction;
1181 state.reviewed_commits = Some(
1182 git::subjects(repo, &start, branch)
1183 .await
1184 .unwrap_or_default(),
1185 );
1186
1187 for id in &takeover.earlier {
1190 if let Ok(old) = RunState::load_under(id, &takeover.home)
1191 && old.repo == state.repo
1192 && (old.pr_head.as_deref() == Some(branch)
1193 || old.winner().is_some_and(|w| w.branch == branch))
1194 {
1195 for hold in old.branch_owner_holds {
1196 if !state.branch_owner_holds.contains(&hold) {
1197 state.event(
1198 "approval-policy",
1199 format!(
1200 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201 hold.control.key(),
1202 hold.file,
1203 hold.head
1204 ),
1205 );
1206 state.branch_owner_holds.push(hold);
1207 }
1208 }
1209 }
1210 }
1211 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215 let tracking = format!("{}/{branch}", state.config.merge.remote);
1216 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219 let mut head = checkout.synced_head.clone();
1220 if matches!(fetched, Ok(out) if out.ok()) {
1221 let remote = git::rev_parse(repo, &tracking).await?;
1222 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223 head = remote;
1224 }
1225 }
1226 head
1227 } else {
1228 git::rev_parse(repo, branch).await?
1229 };
1230 state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232 let worktree = state.worktree_root().join("under-review");
1235 if let Some(parent) = worktree.parent() {
1236 tokio::fs::create_dir_all(parent).await.ok();
1237 }
1238 let path = worktree.to_string_lossy().to_string();
1239 let review_branch = if state.foreign_checkout.is_some() {
1240 let shadow = format!("magi/{}/review", state.short());
1241 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242 shadow
1243 } else {
1244 branch.to_owned()
1245 };
1246 startup_stage(state, "review-checkout")?;
1247 git::git(repo, &["worktree", "add", &path, &review_branch])
1248 .await
1249 .with_context(|| {
1250 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251 })?;
1252
1253 startup_stage(state, "worktree-setup")?;
1254 let setup = match ensure_setup_config(state, repo).await {
1255 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256 Err(e) => Err(e),
1257 };
1258 if let Err(e) = setup {
1259 git::worktree_remove_clean(repo, &worktree).await.ok();
1260 return Err(e);
1261 }
1262 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263 .await
1264 .unwrap_or(0);
1265 if commits == 0 {
1266 git::worktree_remove_clean(repo, &worktree).await.ok();
1267 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268 }
1269 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270 .await
1271 .map(|f| f.len())
1272 .unwrap_or(0);
1273 if files == 0
1274 && let (Ok(head_tree), Ok(base_tree)) = (
1275 git::tree_of(&worktree, "HEAD").await,
1276 git::tree_of(&worktree, &base_commit).await,
1277 )
1278 && head_tree == base_tree
1279 {
1280 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281 git::worktree_remove_clean(repo, &worktree).await.ok();
1282 bail!(
1283 "`{branch}` at {} has a tree identical to base {}; this usually means \
1284 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285 against `{}/{branch}`) rather than an empty change",
1286 short(&head),
1287 short(&base_commit),
1288 state.config.merge.remote
1289 );
1290 }
1291 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292 .await
1293 .unwrap_or_default();
1294
1295 state.candidates.push(Candidate {
1296 index: 0,
1297 label: 'A',
1298 agent: EXISTING_BRANCH.to_owned(),
1301 branch: review_branch,
1302 worktree,
1303 summary: String::new(),
1304 stat,
1305 files,
1306 commits,
1307 empty: false,
1308 failed: None,
1309 verified_noop: None,
1310 duration_ms: 0,
1311 folded: false,
1312 });
1313 state.tally = Some(Tally {
1314 first_choice: BTreeMap::from([('A', 0)]),
1315 borda: BTreeMap::new(),
1316 winner: 'A',
1317 rankings: 0,
1318 unanimous_initial: false,
1319 deliberated: false,
1320 changed_votes: 0,
1321 unanimous_final: false,
1322 tie_break: None,
1323 judges: 0,
1327 present: 0,
1328 quorum: 0,
1329 met_quorum: true,
1330 uncontested: Some("review-only run: nothing competed".to_owned()),
1331 });
1332 state.status = RunStatus::Reviewing;
1333 state.prep_complete = Some(true);
1334 state.event(
1335 "start",
1336 format!(
1337 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338 state.id
1339 ),
1340 );
1341 Ok(())
1342 }
1343
1344 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347 Self::authorize_base_recovery_request(home, id, task, false).await
1348 }
1349
1350 pub(crate) async fn authorize_base_recovery_from_resumption(
1352 home: &Path,
1353 id: &str,
1354 task: &str,
1355 ) -> Result<()> {
1356 Self::authorize_base_recovery_request(home, id, task, true).await
1357 }
1358
1359 async fn authorize_base_recovery_request(
1360 home: &Path,
1361 id: &str,
1362 task: &str,
1363 resumption: bool,
1364 ) -> Result<()> {
1365 let _owner = crate::pr_head::owner_lock(home, id)?;
1366 let mut state = RunState::load_under(id, home)?;
1367 crate::pr_head::ensure_owner(&state)?;
1368 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369 anyhow::ensure!(
1370 queued.runs.last().is_some_and(|run| run == id)
1371 && queued.status != crate::queue::TaskStatus::Done
1372 && !queued.fresh_start
1373 && queued.review_branch.is_none(),
1374 "run task ownership changed"
1375 );
1376 anyhow::ensure!(
1377 state
1378 .origin
1379 .as_ref()
1380 .and_then(|o| o.task.as_deref())
1381 .is_none_or(|owner| owner == task),
1382 "run belongs to another task"
1383 );
1384 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385 anyhow::ensure!(
1386 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387 "run is live or its ownership is unknown"
1388 );
1389 let _execution = crate::driver::maintenance(home, id)?;
1392 anyhow::ensure!(
1393 state.status.resumable()
1394 && state.released_to.is_none()
1395 && state.retired_to.is_none()
1396 && state.pr_taken_over_by.is_none(),
1397 "run ownership is final"
1398 );
1399 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400 anyhow::ensure!(
1401 !state.pr_publication_pending
1402 && !crate::handover::landing_checkpoint(&state)
1403 && state.merge_decision.is_none(),
1404 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405 );
1406 let winner = state.winner().context("run has no retained winner")?;
1407 anyhow::ensure!(
1408 git::is_clean(&winner.worktree).await?,
1409 "retained worktree has uncommitted changes"
1410 );
1411 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412 anyhow::ensure!(
1413 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414 "retained worktree does not match the winner branch"
1415 );
1416 state.record_base_sync_stop(&head);
1417 state.save()?;
1418 if resumption {
1419 anyhow::ensure!(
1420 crate::daemon::compatible_run_instruction(&state, &queued),
1421 "resumption instruction does not match the retained run"
1422 );
1423 let approval = queued
1424 .resumption
1425 .as_ref()
1426 .context("no operator resumption")?;
1427 anyhow::ensure!(
1428 approval.instruction == queued.instruction,
1429 "resumption approval does not match the current task instruction"
1430 );
1431 state.authorize_base_sync_resumption(task, approval)?;
1432 } else {
1433 state.authorize_base_sync(task)?;
1434 }
1435 state.event(
1436 "base-recovery",
1437 "owner authorized one additional base rebase; fresh verification required",
1438 );
1439 state.save()?;
1440 Ok(())
1441 }
1442
1443 pub fn resume(id: &str) -> Result<Self> {
1445 let state = RunState::load(id)?;
1446 if state.startup.as_ref().is_some_and(|s| !s.complete)
1447 || (state.status == RunStatus::Prep
1448 && state.instruction.trim().is_empty()
1449 && state.candidates.is_empty()
1450 && state.prep_complete == Some(false))
1451 {
1452 bail!(
1453 "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1454 );
1455 }
1456 crate::pr_head::ensure_owner(&state)?;
1457 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1458 bail!(
1459 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1460 );
1461 }
1462 if let Some(to) = &state.released_to {
1463 bail!(
1464 "run {} cannot be resumed: its worktree was released to run {}",
1465 state.short(),
1466 crate::run::short_of(to)
1467 );
1468 }
1469 let roles = state.config.resolve_roles()?;
1470 if let Some(id) = &state.config.graph.selected_implementer
1471 && state.candidates.iter().any(|c| c.agent != *id)
1472 {
1473 bail!(
1474 "replacement implementer pin does not match the saved candidates; task must stay held"
1475 );
1476 }
1477 let max_parallel = state.config.graph.max_parallel.max(1);
1478 Ok(Self {
1479 state,
1480 roles,
1481 sem: Arc::new(Semaphore::new(max_parallel)),
1482 pause: Pause::new(),
1483 interrupt: Pause::new(),
1484 })
1485 }
1486
1487 pub async fn execute(&mut self) -> Result<()> {
1494 anyhow::ensure!(
1495 self.state.startup.as_ref().is_none_or(|s| s.complete),
1496 "startup incomplete; inspect retained evidence before explicit recovery"
1497 );
1498 if let Some(selection) = &self.state.pr_recovery
1499 && (selection.source_proof.is_some() || selection.source_run.is_some())
1500 {
1501 Box::pin(selection.verify_source(&self.state.repo)).await?;
1502 }
1503 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1506 || self.state.land_approval.is_some()
1507 || self.state.merge_decision.is_some()
1508 || !self.state.merge_repairs.is_empty()
1509 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1510 {
1511 let claim = FixClaim::acquire(&self.state.dir())?;
1512 let disk = RunState::load(&self.state.id)?;
1513 let checkpoint = |s: &RunState| {
1514 (
1515 s.merge_repairs.len(),
1516 s.merge_repairs
1517 .last()
1518 .map(|r| (r.question.clone(), r.phase)),
1519 s.land_approval
1520 .as_ref()
1521 .map(|a| (a.question.clone(), a.head.clone())),
1522 )
1523 };
1524 if checkpoint(&disk) != checkpoint(&self.state) {
1525 bail!(
1526 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1527 );
1528 }
1529 Some(claim)
1530 } else {
1531 None
1532 };
1533 let mut execution = self.claim_driver().await?;
1534 let runner = &mut *execution.runner;
1535 let wait = runner.repository_wait();
1536 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1539 let result = match result {
1540 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1541 record_repository_block(&mut runner.state, &e);
1542 runner.park_here()?;
1543 Ok(())
1544 }
1545 other => other,
1546 };
1547 execution.driver.finish(&mut runner.state).await?;
1548 let ended = if result.is_err() {
1549 Some({
1550 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1551 if let Err(error) = &result {
1552 notice.message = format!(
1553 "Run {} stopped with an error: {}",
1554 runner.state.id,
1555 error.to_string().lines().next().unwrap_or("unknown error")
1556 );
1557 }
1558 notice
1559 })
1560 } else {
1561 crate::notices::run_ended(&runner.state)
1562 };
1563 if let Some(notice) = ended {
1564 crate::notices::raise(notice);
1565 }
1566 result
1567 }
1568
1569 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1570 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1571 }
1572
1573 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1574 let wait = self.repository_wait();
1575 coordinate_operation(&mut self.state, cwd, &wait).await
1576 }
1577
1578 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1580 let owner = match crate::pr_head::owner_lock_wait(
1581 &crate::run::home(),
1582 &self.state.id,
1583 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1584 )
1585 .await
1586 {
1587 Ok(owner) => owner,
1588 Err(error) => {
1589 let detail = format!("worktree maintenance or ownership update: {error:#}");
1590 self.state.resource_block = Some(crate::run::ResourceBlock {
1594 cause: crate::run::ResourceBlockCause::Other,
1595 node: "start".into(),
1596 kind: "worktree maintenance".into(),
1597 detail: detail.clone(),
1598 at: Timestamp::now(),
1599 });
1600 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1601 }
1602 };
1603 let disk = RunState::load(&self.state.id)?;
1604 crate::pr_head::ensure_owner(&disk)?;
1605 let claimed =
1606 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1607 anyhow::ensure!(
1612 disk.liveness(claimed) != crate::run::Liveness::Live,
1613 "run already has a live driver; wait for its execution to end"
1614 );
1615 if let Some(to) = &disk.released_to {
1616 bail!(
1617 "run {} cannot continue: its worktree was released to run {}",
1618 self.state.short(),
1619 crate::run::short_of(to)
1620 );
1621 }
1622 anyhow::ensure!(
1623 serde_json::to_value(&disk.base_sync_recovery)?
1624 == serde_json::to_value(&self.state.base_sync_recovery)?,
1625 "base recovery changed since this run was loaded; reload the saved run"
1626 );
1627 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1628 let mut execution = DriverExecution {
1629 runner: self,
1630 driver,
1631 };
1632 let _owner = owner;
1635 let state = &mut execution.runner.state;
1636 state.parked = false;
1641 state.resource_wait = None;
1642 state.resource_block = None;
1643 state.clear_active();
1650 let pid = std::process::id();
1665 state.driver_pid = Some(pid);
1666 state.driver_started_at = crate::proc::process_started_at(pid);
1667 state.driver_exited = false;
1668 state.driver_claimed = Some(true);
1669 state.begin_pass();
1670 execution.driver.bind(state);
1671 state.save()?;
1672 execution.driver.record(state)?;
1673 Ok(execution)
1674 }
1675
1676 async fn execute_graph(&mut self) -> Result<()> {
1677 self.state.configure_recovery_resume()?;
1678 self.persist_candidates().await?;
1681 if self.state.base_sync_blocked() {
1684 self.sync_to_base().await?;
1685 if !self.state.base_sync_blocked()
1686 && self.state.status != RunStatus::AlreadyInBase
1687 && !self.resource_halt()?
1688 {
1689 self.finish_after_tally().await?;
1690 }
1691 return Ok(());
1692 }
1693 if self
1694 .state
1695 .merge_decision
1696 .as_ref()
1697 .is_some_and(|decision| !decision.closed)
1698 && self.state.status != RunStatus::Landing
1699 {
1700 if crate::merge_decision::resumable_hold(&self.state) {
1701 self.state.merge = Some(MergeOutcome {
1702 mode: MergeMode::Pr,
1703 ok: false,
1704 detail: self
1705 .state
1706 .merge_decision
1707 .as_ref()
1708 .unwrap()
1709 .target
1710 .pr
1711 .clone(),
1712 empty: false,
1713 });
1714 self.run_land().await?;
1715 self.settle_questions();
1716 return Ok(());
1717 }
1718 bail!(
1719 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1720 );
1721 }
1722 if self.state.config.graph.selected_implementer.is_some()
1723 && self.state.status == RunStatus::Blocked
1724 && self.state.tally.is_none()
1725 && self.state.candidate_preservation.is_empty()
1726 && self.state.interrupted_implementations.is_empty()
1727 {
1728 bail!(
1729 "selected implementation is held; explicitly release the owning task to retry it"
1730 );
1731 }
1732
1733 if self.state.merge_repairs.last().is_some_and(|r| {
1734 matches!(
1735 r.phase,
1736 crate::run::MergeRepairPhase::Requested
1737 | crate::run::MergeRepairPhase::Fixing
1738 | crate::run::MergeRepairPhase::Reviewing
1739 )
1740 }) {
1741 if self.repair_merge_review().await? {
1742 self.run_land().await?;
1743 }
1744 self.settle_questions();
1745 return Ok(());
1746 }
1747 if self.state.pr_publication_pending {
1750 self.merge().await?;
1751 if self.resource_halt()? {
1752 return Ok(());
1753 }
1754 self.state.save()?;
1755 self.settle_questions();
1756 return Ok(());
1757 }
1758 if self.state.status == RunStatus::Stalled {
1771 if self.recover_stall().await? {
1772 self.finish_after_tally().await?;
1773 } else {
1774 self.state.save()?;
1776 }
1777 return Ok(());
1778 }
1779 if self.state.status == RunStatus::Landing
1789 || crate::handover::landing_checkpoint(&self.state)
1790 {
1791 self.state.land_sync_pending = true;
1792 self.state.save()?;
1793 self.run_land().await?;
1794 self.settle_questions();
1799 return Ok(());
1800 }
1801 if self.state.github_text.is_some()
1806 && self.state.merge.is_none()
1807 && self.state.status == RunStatus::Gating
1808 {
1809 self.merge().await?;
1810 if self.resource_halt()? {
1811 return Ok(());
1812 }
1813 self.state.save()?;
1814 return Ok(());
1815 }
1816 if let Err(e) = self.prep().await {
1817 if e.is::<crate::repo_lock::ResourceBlocked>() {
1818 return Err(e);
1819 }
1820 self.state.status = RunStatus::Blocked;
1821 self.state
1822 .prep_failure
1823 .get_or_insert_with(|| format!("{e:#}"));
1824 self.state
1825 .event("prep", format!("preparation stopped: {e:#}"));
1826 self.state.save()?;
1827 return Err(e);
1828 }
1829 if self.park_here()? {
1830 return Ok(());
1831 }
1832 self.advise().await?;
1833 if self.park_here()? {
1834 return Ok(());
1835 }
1836 self.implement().await?;
1837 if self.park_here()? {
1838 return Ok(());
1839 }
1840 if self.state.status == RunStatus::VerifiedNoop {
1844 return Ok(());
1845 }
1846 self.judge().await?;
1847 if self.park_here()? {
1848 return Ok(());
1849 }
1850 self.deliberate().await?;
1851 if self.park_here()? {
1852 return Ok(());
1853 }
1854 self.vote().await?;
1855 if self.park_here()? {
1856 return Ok(());
1857 }
1858 self.tally()?;
1859 if self.state.status == RunStatus::Stalled {
1864 self.state.save()?;
1868 return Ok(());
1869 }
1870 self.finish_after_tally().await?;
1871 Ok(())
1872 }
1873
1874 fn stop_for_resource(&mut self) -> Result<()> {
1879 if self.pause.parked() || self.interrupt.parked() {
1882 self.park_here()?;
1883 return Ok(());
1884 }
1885 if self.state.resource_block.is_some() {
1886 self.state.status = RunStatus::Blocked;
1887 }
1888 self.state.save()
1889 }
1890
1891 fn resource_halt(&mut self) -> Result<bool> {
1893 if self.state.resource_block.is_none() {
1894 return Ok(false);
1895 }
1896 self.stop_for_resource()?;
1897 Ok(true)
1898 }
1899
1900 fn park_here(&mut self) -> Result<bool> {
1907 let parking = self.pause.parked() || self.interrupt.parked();
1908 if !parking && self.state.resource_block.is_some() {
1909 self.stop_for_resource()?;
1910 return Ok(true);
1911 }
1912 if !self.pause.parked() && !self.interrupt.parked() {
1917 return Ok(false);
1918 }
1919 if self.state.parked {
1920 return Ok(true);
1921 }
1922 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1923 if let Some(block) = &mut self.state.resource_block
1924 && matches!(
1925 block.cause,
1926 crate::run::ResourceBlockCause::Cancelled { .. }
1927 )
1928 {
1929 block.cause = crate::run::ResourceBlockCause::Cancelled {
1930 park: cause.clone(),
1931 };
1932 }
1933 let why = match cause.as_ref().map(ParkCause::label) {
1934 Some(reason) => format!(
1935 "parked after `{}` ({reason}) — resume to carry on from here",
1936 self.state.status.as_str()
1937 ),
1938 None => format!(
1939 "parked after `{}` — resume to carry on from here",
1940 self.state.status.as_str()
1941 ),
1942 };
1943 self.state.event("park", why);
1944 self.state
1945 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1946 self.state.parked = true;
1947 self.state.save()?;
1948 Ok(true)
1949 }
1950
1951 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1953 if self.state.prep_failure.is_none() {
1954 return Ok(());
1955 }
1956 let failed_at = self
1957 .state
1958 .events
1959 .iter()
1960 .rev()
1961 .find(|event| event.node == "prep")
1962 .map_or(self.state.updated_at, |event| event.at);
1963 anyhow::ensure!(
1964 approval_at > failed_at,
1965 "preparation retry requires a new explicit release after the failure"
1966 );
1967 self.state.prep_failure = None;
1968 self.state.prep_complete = Some(false);
1969 self.state
1970 .event("prep-retry", "operator approved retry of preparation");
1971 Ok(())
1972 }
1973
1974 pub fn on_pause(&mut self, pause: Pause) {
1976 self.pause = pause;
1977 }
1978
1979 pub fn watch_interrupt(&mut self, pause: Pause) {
1985 self.interrupt = pause;
1986 }
1987
1988 fn settle_questions(&mut self) {
2008 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2009 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2010 }
2011 }
2012
2013 async fn finish_after_tally(&mut self) -> Result<()> {
2016 self.graft().await?;
2019 if self.resource_halt()? {
2020 return Ok(());
2021 }
2022 self.fold_losers().await?;
2023 self.sync_to_base().await?;
2028 if self.hold_base_sync().await? || self.resource_halt()? {
2029 return Ok(());
2030 }
2031 if self.state.status == RunStatus::AlreadyInBase {
2032 return Ok(());
2033 }
2034 self.review_loop().await?;
2035 if self.resource_halt()? {
2036 return Ok(());
2037 }
2038 self.sync_to_base().await?;
2039 if self.hold_base_sync().await? || self.resource_halt()? {
2040 return Ok(());
2041 }
2042 if self.state.status == RunStatus::AlreadyInBase {
2043 return Ok(());
2044 }
2045 self.gate().await?;
2046 self.merge().await?;
2047 if self.resource_halt()? {
2048 return Ok(());
2049 }
2050 self.state.save()?;
2051 Ok(())
2052 }
2053
2054 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2061 let result = match ensure_setup_config(&mut self.state, repo).await {
2062 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2063 Err(e) => Err(e),
2064 };
2065 if let Err(e) = result {
2066 git::worktree_remove_clean(repo, wt).await.ok();
2067 if in_prep {
2068 self.rollback_prep(repo).await;
2069 }
2070 self.state.status = RunStatus::Blocked;
2071 self.state.event(
2072 "setup",
2073 format!("worktree setup failed in {}: {e:#}", wt.display()),
2074 );
2075 self.state.save()?;
2076 return Err(e);
2077 }
2078 Ok(())
2079 }
2080
2081 async fn rollback_prep(&mut self, repo: &Path) {
2083 let root = self.state.worktree_root();
2084 for c in std::mem::take(&mut self.state.candidates) {
2085 if matches!(
2086 git::worktree_remove_clean(repo, &c.worktree).await,
2087 Ok(true)
2088 ) {
2089 git::branch_delete(repo, &c.branch).await.ok();
2090 } else {
2091 self.state.candidates.push(c);
2092 }
2093 }
2094 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2095 for j in 1..=seats {
2096 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2097 .await
2098 .ok();
2099 }
2100 for k in 1..=self.state.config.graph.advisors {
2101 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2102 .await
2103 .ok();
2104 }
2105 }
2106
2107 async fn prep(&mut self) -> Result<()> {
2108 if let Some(reason) = &self.state.prep_failure {
2109 bail!(
2110 "candidate preparation previously failed: {reason}; correct the source before retrying"
2111 );
2112 }
2113 if self
2114 .state
2115 .prep_complete
2116 .unwrap_or(!self.state.candidates.is_empty())
2117 {
2118 return Ok(());
2119 }
2120 let repo = self.state.repo.clone();
2121 let _coordination = self.coordinate(&repo).await?;
2124 let base = self.state.base_commit.clone();
2125 let plan = self.state.sources.plan(&repo, &base).await?;
2126 self.rollback_prep(&repo).await;
2127 if !self.state.candidates.is_empty() {
2128 bail!(
2129 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2130 );
2131 }
2132 self.state.prep_complete = Some(false);
2133 self.state.status = RunStatus::Prep;
2134 self.state.save()?;
2135 let repo = self.state.repo.clone();
2136 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2137 let root = self.state.worktree_root();
2138 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2139
2140 let hooks_dir = self.state.dir().join("hooks");
2143 if self.state.config.blind.commit_msg_hook {
2144 std::fs::create_dir_all(&hooks_dir)
2145 .with_context(|| format!("create {}", hooks_dir.display()))?;
2146 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2147 let path = hooks_dir.join("commit-msg");
2148 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2149 make_executable(&path)?;
2150 git::acquire_worktree_config(&repo).await?;
2158 self.state.enabled_worktree_config = true;
2159 }
2160
2161 ensure_setup_config(&mut self.state, &repo).await?;
2164
2165 for (index, (spec, label)) in self
2166 .roles
2167 .implementers
2168 .clone()
2169 .into_iter()
2170 .zip(labels)
2171 .enumerate()
2172 {
2173 let branch = self.state.branch_for(label);
2174 let worktree = root.join(format!("cand-{label}"));
2175 self.state.event(
2179 "prep-intent",
2180 serde_json::json!({
2181 "branch": branch,
2182 "worktree": worktree,
2183 "start": start,
2184 "picks": plan.picks,
2185 })
2186 .to_string(),
2187 );
2188 self.state.save()?;
2189 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2190 if self.state.config.blind.commit_msg_hook {
2191 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2192 }
2193 git::local_exclude(&worktree, "/.magi/").await?;
2194 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2195 git::worktree_remove_clean(&repo, &worktree).await.ok();
2198 git::branch_delete(&repo, &branch).await.ok();
2199 self.rollback_prep(&repo).await;
2200 self.state.status = RunStatus::Blocked;
2201 self.state
2202 .event("prep", format!("worktree setup failed: {e:#}"));
2203 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2204 self.state.save()?;
2205 return Err(e);
2206 }
2207 self.state.candidates.push(Candidate {
2208 index,
2209 label,
2210 agent: spec.id.clone(),
2211 branch,
2212 worktree,
2213 summary: String::new(),
2214 stat: String::new(),
2215 files: 0,
2216 commits: 0,
2217 empty: false,
2218 failed: None,
2219 verified_noop: None,
2220 duration_ms: 0,
2221 folded: false,
2222 });
2223 self.state.save()?;
2224 for pick in &plan.picks {
2225 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2226 {
2227 self.state.status = RunStatus::Blocked;
2228 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2229 self.state
2230 .event("prep", format!("cannot apply referenced commit: {e}"));
2231 self.state.save()?;
2232 return Err(e);
2233 }
2234 }
2235 }
2236
2237 for j in 1..=self.roles.judges.len() {
2238 let wt = root.join(format!("judge-{j}"));
2239 if !wt.exists() {
2240 git::worktree_add_detached(&repo, &wt, &base).await?;
2241 self.setup_seat_worktree(&repo, &wt, true).await?;
2242 }
2243 }
2244
2245 if self.state.config.graph.advise {
2253 for k in 1..=self.state.config.graph.advisors {
2254 let wt = root.join(format!("advisor-{k}"));
2255 if !wt.exists() {
2256 git::worktree_add_detached(&repo, &wt, &base).await?;
2257 self.setup_seat_worktree(&repo, &wt, true).await?;
2258 }
2259 }
2260 }
2261
2262 let authors: Vec<&str> = self
2267 .roles
2268 .implementers
2269 .iter()
2270 .map(|a| a.id.as_str())
2271 .collect();
2272 let overlap: Vec<String> = self
2273 .roles
2274 .judges
2275 .iter()
2276 .enumerate()
2277 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2278 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2279 .collect();
2280 if !overlap.is_empty() {
2281 let note = format!(
2282 "{} also authored a candidate; blind, but the panel is less \
2283 independent than {} distinct agents would be",
2284 overlap.join(", "),
2285 self.roles.judges.len()
2286 );
2287 self.state.event("prep", note);
2288 }
2289
2290 self.state.event(
2291 "prep",
2292 format!(
2293 "{} candidates, {} judges, base {} ({})",
2294 self.state.candidates.len(),
2295 self.roles.judges.len(),
2296 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2297 self.state.base_branch
2298 ),
2299 );
2300 self.state.prep_complete = Some(true);
2301 self.state.status = RunStatus::Implementing;
2302 self.state.save()?;
2303 Ok(())
2304 }
2305
2306 async fn advise(&mut self) -> Result<()> {
2339 let implement_untouched = self
2340 .state
2341 .candidates
2342 .iter()
2343 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2344 if !self.state.config.graph.advise || self.state.advise_attempted {
2345 return Ok(());
2346 }
2347 if !implement_untouched {
2348 self.state.event(
2349 "advise",
2350 "skipping the design-deliberation stage: at least one \
2351 candidate already shows implementation progress, so this \
2352 run is past the point the stage exists to run before"
2353 .to_owned(),
2354 );
2355 self.state.advise_attempted = true;
2356 self.state.save()?;
2357 return Ok(());
2358 }
2359 let run_id = self.state.id.clone();
2360 let prompts = self.state.config.prompts.clone();
2361 let instruction = self.state.instruction.clone();
2362 let language = self.state.config.graph.language.clone();
2363 let root = self.state.worktree_root();
2364 let n = self.state.config.graph.advisors;
2365 let where_recorded = self.state.dir().join("run.json");
2366
2367 let seats = match self.state.config.advisors() {
2368 Ok(seats) if !seats.is_empty() => seats,
2369 Ok(_) => {
2370 self.state.event(
2371 "advise",
2372 format!(
2373 "[graph] advisors is 0; skipping the design-deliberation \
2374 stage and continuing without a synthesis brief (see {})",
2375 where_recorded.display()
2376 ),
2377 );
2378 self.state.advise_attempted = true;
2379 self.state.save()?;
2380 return Ok(());
2381 }
2382 Err(e) => {
2383 self.state.event(
2384 "advise",
2385 format!(
2386 "could not resolve advisor seats ({e:#}); continuing \
2387 without a design-deliberation brief (see {})",
2388 where_recorded.display()
2389 ),
2390 );
2391 self.state.advise_attempted = true;
2392 self.state.save()?;
2393 return Ok(());
2394 }
2395 };
2396
2397 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2398 let artifacts = agent::artifacts_dir(&self.state.dir());
2399 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2400
2401 let mut jobs = Vec::new();
2402 for (i, spec) in seats.iter().cloned().enumerate() {
2403 let seat_key = format!("advisor-{}", i + 1);
2404 let seat = self.seat(&seat_key, &spec.id);
2405 jobs.push(SeatJob {
2406 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2407 spec,
2408 seat,
2409 cwd: worktrees[i % worktrees.len()].clone(),
2410 timeout,
2411 allow_write: false,
2412 sessions: false,
2413 artifacts: artifacts.clone(),
2414 stem: seat_key,
2415 handover: None,
2416 });
2417 }
2418
2419 self.state.event(
2420 "advise",
2421 format!(
2422 "{} advisor seat(s) sketching a design in parallel",
2423 jobs.len()
2424 ),
2425 );
2426 let mut quota_losses = Vec::new();
2427 let cache = self.state.config.cache_dir();
2428 let ctx = WaveCtx {
2429 carry_seats: false,
2430 pauses: Some((&self.pause, &self.interrupt)),
2431 run: &run_id,
2432 node: "advise",
2433 prompts: &prompts,
2434 cache: cache.as_deref(),
2435 round: None,
2436 };
2437 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2438 let results = ask_json_wave::<Proposal>(
2439 jobs,
2440 Arc::clone(&self.sem),
2441 self.state.config.graph.retries,
2442 &advisor_roster,
2443 &ctx,
2444 &mut quota_losses,
2445 &mut self.state,
2446 &|p: &Proposal| p.validate(),
2447 )
2448 .await;
2449 if self.state.resource_block.is_some() {
2450 return self.stop_for_resource();
2451 }
2452 self.state.quota.extend(quota_losses);
2453
2454 let mut records = Vec::with_capacity(results.len());
2455 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2456 let agent_id = seat.agent.clone();
2457 self.state.seats.insert(seat.key.clone(), seat);
2458 match res {
2459 Ok((proposal, out)) => {
2460 self.state
2461 .event("advise", format!("advisor-{} proposed a design", i + 1));
2462 records.push(advise::AdvisorRecord::proposed(
2463 i + 1,
2464 agent_id,
2465 proposal,
2466 out.duration_ms,
2467 ));
2468 }
2469 Err(e) => {
2470 self.state.event(
2471 "advise",
2472 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2473 );
2474 records.push(advise::AdvisorRecord::failed(
2475 i + 1,
2476 agent_id,
2477 e.to_string(),
2478 ));
2479 }
2480 }
2481 }
2482
2483 let mut advice = advise::Advice {
2484 records,
2485 synthesis: None,
2486 };
2487 if advice.proposals().is_empty() {
2488 self.state.event(
2489 "advise",
2490 "no advisor produced a usable proposal; continuing without a \
2491 synthesis brief"
2492 .to_owned(),
2493 );
2494 } else {
2495 match self
2496 .synthesize_brief(
2497 &advice,
2498 &instruction,
2499 &language,
2500 &worktrees[0],
2501 &artifacts,
2502 &run_id,
2503 &prompts,
2504 cache.as_deref(),
2505 )
2506 .await
2507 {
2508 Ok(Some(text)) => {
2509 self.state.event(
2510 "advise",
2511 "synthesized a design brief for the implementer".to_owned(),
2512 );
2513 advice.synthesis = Some(text);
2514 }
2515 Ok(None) => {
2516 self.state.event(
2517 "advise",
2518 "the synthesis seat produced nothing usable; continuing \
2519 without a design brief"
2520 .to_owned(),
2521 );
2522 }
2523 Err(e) => {
2524 if e.is::<crate::repo_lock::ResourceBlocked>() {
2525 return Err(e);
2526 }
2527 self.state.event(
2528 "advise",
2529 format!("could not synthesize a design brief: {e:#}"),
2530 );
2531 }
2532 }
2533 }
2534 advise::apply_reflection(&mut advice);
2535
2536 self.state.advice = Some(advice);
2537 self.state.advise_attempted = true;
2538 self.state.save()?;
2539 Ok(())
2540 }
2541
2542 #[allow(clippy::too_many_arguments)]
2554 async fn synthesize_brief(
2555 &mut self,
2556 advice: &advise::Advice,
2557 instruction: &str,
2558 language: &str,
2559 cwd: &Path,
2560 artifacts: &Path,
2561 run_id: &str,
2562 prompts: &Prompts,
2563 cache: Option<&Path>,
2564 ) -> Result<Option<String>> {
2565 let chain = agent::pick_chain(
2566 &self.state.config.agents,
2567 self.state.config.roles.synthesizer.as_ref(),
2568 &agent::installed,
2569 "synthesizer",
2570 )?;
2571 let proposals = advice.proposals();
2572 let mut prompt = prompt::with_overlay(
2573 prompt::synthesize_brief(instruction, &proposals, language),
2574 prompts.overlay("advise"),
2575 );
2576 if cache.is_some() {
2577 prompt.push('\n');
2582 prompt.push_str(&prompt::build_cache_note("advise", false));
2583 }
2584 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2585 let mut last = None;
2590 for (n, spec) in chain.iter().enumerate() {
2591 if n > 0 {
2592 self.state
2593 .event("advise", format!("synthesis falling back to {}", spec.id));
2594 }
2595 let mut seat = self.seat("advise-synthesis", &spec.id);
2596 let wait = self.repository_wait();
2597 let outcome = wait
2598 .scope(agent::invoke(
2599 spec,
2600 &mut seat,
2601 &Invocation {
2602 coordination: self.state.config.coordination.as_ref(),
2603 fetched_base: None,
2604 cwd,
2605 prompt: &prompt,
2606 timeout,
2607 allow_write: false,
2608 sessions: false,
2609 artifacts,
2610 stem: &if n == 0 {
2611 "advise-synthesis".to_owned()
2612 } else {
2613 format!("advise-synthesis-{}", spec.id)
2614 },
2615 run: run_id,
2616 node: "advise",
2617 cache_dir: None,
2618 attachments: &[],
2619 writable: &[],
2620 },
2621 ))
2622 .await;
2623 if outcome.is_ok() {
2624 self.state.seats.insert(seat.key.clone(), seat);
2625 }
2626 let advance = agent::chain_advances(&outcome);
2627 last = Some(outcome);
2628 if !advance {
2629 break;
2630 }
2631 }
2632 let out = last.expect("a chain holds at least one agent")?;
2635 if !out.usable() {
2636 return Ok(None);
2637 }
2638 let text =
2639 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2640 Ok((!text.trim().is_empty()).then_some(text))
2641 }
2642
2643 async fn implement(&mut self) -> Result<()> {
2646 self.persist_candidates().await?;
2647 let run_id = self.state.id.clone();
2652 let prompts = self.state.config.prompts.clone();
2653 let todo: Vec<usize> = self
2654 .state
2655 .candidates
2656 .iter()
2657 .enumerate()
2658 .filter(|(_, c)| {
2659 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2660 && c.failed.is_none()
2661 && !c.empty
2662 })
2663 .map(|(i, _)| i)
2664 .collect();
2665 if todo.is_empty() {
2666 return self.after_implement();
2667 }
2668 self.state.status = RunStatus::Implementing;
2669
2670 let language = self.state.config.graph.language.clone();
2671 let github_language = self.state.config.graph.github_language.clone();
2672 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2673 let sessions = self.state.config.graph.sessions;
2674 let artifacts = agent::artifacts_dir(&self.state.dir());
2675 let brief = self
2679 .state
2680 .advice
2681 .as_ref()
2682 .and_then(|a| a.synthesis.as_deref())
2683 .map(str::to_owned);
2684 let attachments = self.state.attachments.clone();
2685
2686 let mut jobs = Vec::new();
2687 for &i in &todo {
2688 let (index, label, worktree) = {
2689 let c = &self.state.candidates[i];
2690 (c.index, c.label, c.worktree.clone())
2691 };
2692 let spec = self.roles.implementers[index].clone();
2693 let seat_key = format!("impl-{label}");
2694 let seat = self.seat(&seat_key, &spec.id);
2695 let instruction = seeded_instruction(&self.state);
2696 jobs.push(SeatJob {
2697 spec,
2698 seat,
2699 prompt: prompt::implement_in(
2700 &instruction,
2701 &worktree.to_string_lossy(),
2702 &language,
2703 &github_language,
2704 brief.as_deref(),
2705 &attachments,
2706 ),
2707 cwd: worktree,
2708 timeout,
2709 allow_write: true,
2710 sessions,
2711 artifacts: artifacts.clone(),
2712 stem: format!("impl-{label}"),
2713 handover: None,
2714 });
2715 }
2716
2717 self.state.event(
2718 "implement",
2719 format!("{} candidates in parallel", jobs.len()),
2720 );
2721 let mut sent = jobs.clone();
2727 let cache = self.state.config.cache_dir();
2728 let ctx = WaveCtx {
2729 carry_seats: false,
2730 pauses: Some((&self.pause, &self.interrupt)),
2731 run: &run_id,
2732 node: "implement",
2733 prompts: &prompts,
2734 cache: cache.as_deref(),
2735 round: None,
2736 };
2737 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2738 if !wave_blocked(&results) {
2739 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2740 .await;
2741 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2742 .await?;
2743 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2744 .await;
2745 }
2746 if !results.is_empty()
2749 && results
2750 .iter()
2751 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
2752 {
2753 let losses: Vec<_> = results
2754 .iter()
2755 .map(|(_, seat, out)| {
2756 let AgentOutcome::Quota(o) = out else {
2757 unreachable!()
2758 };
2759 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
2760 })
2761 .collect();
2762 self.state.quota.extend(losses.clone());
2763 for (_, seat, _) in results {
2764 self.state.seats.insert(seat.key.clone(), seat);
2765 }
2766 for &i in &todo {
2767 let label = self.state.candidates[i].label;
2768 self.state.interrupted_implementations.insert(label);
2769 self.state
2770 .candidate_preservation
2771 .insert(label, "machine outage awaiting persistence".into());
2772 }
2773 block_machine_outage(&mut self.state, "implement", &losses);
2776 self.state.save()?;
2777 self.persist_candidates().await?;
2778 for &i in &todo {
2779 self.state
2780 .interrupted_implementations
2781 .insert(self.state.candidates[i].label);
2782 }
2783 return self.stop_for_resource();
2784 }
2785 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2786
2787 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2788 && results
2789 .iter()
2790 .any(|(_, _, out)| FailClass::of(out).is_some());
2791 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2792 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2793 if self
2797 .state
2798 .candidate_preservation
2799 .contains_key(&self.state.candidates[i].label)
2800 {
2801 self.state.seats.insert(seat.key.clone(), seat);
2802 }
2803 continue;
2804 }
2805 let seat_key = seat.key.clone();
2806 let agent = seat.agent.clone();
2815 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2816 self.state.seats.insert(seat.key.clone(), seat);
2817 let label = self.state.candidates[i].label;
2818 self.state.interrupted_implementations.remove(&label);
2819
2820 let (summary, duration, failed, verified_claim) = match out {
2821 AgentOutcome::Ok(o) => {
2822 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2823 let failed = (!o.usable()).then(|| {
2824 if o.timed_out {
2825 "agent timed out".to_owned()
2826 } else {
2827 format!(
2828 "agent exited with {}",
2829 crate::notices::exit_status(o.exit_code)
2830 )
2831 }
2832 });
2833 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2834 (text, o.duration_ms, failed, verified_claim)
2835 }
2836 AgentOutcome::Dropped(o) => {
2842 let why = o
2843 .dropped
2844 .as_ref()
2845 .map(|d| d.why.as_str())
2846 .unwrap_or("the CLI ended the stream without delivering its answer");
2847 (
2848 String::new(),
2849 o.duration_ms,
2850 Some(format!("the CLI dropped the stream ({why})")),
2851 None,
2852 )
2853 }
2854 AgentOutcome::Quota(o) => {
2855 self.state.quota.push(QuotaLoss::from_report(
2856 seat_key,
2857 "implement",
2858 o.quota.as_ref(),
2859 ));
2860 (
2861 String::new(),
2862 o.duration_ms,
2863 Some("rate limited (quota); produced no change".to_owned()),
2864 None,
2865 )
2866 }
2867 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2868 (String::new(), 0, Some(e), None)
2869 }
2870 };
2871
2872 let c = &mut self.state.candidates[i];
2875 if !exhausted_the_fallback_chain {
2876 c.agent = agent;
2877 }
2878 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2879 c.duration_ms = duration;
2880 c.failed = failed;
2881 c.verified_noop = verified_claim;
2882 self.state
2883 .candidate_preservation
2884 .insert(label, "awaiting persistence".to_owned());
2885 }
2886 self.state.save()?;
2887 self.persist_candidates().await?;
2888
2889 if resource_blocked {
2890 return self.stop_for_resource();
2891 }
2892 if selected_failed {
2893 self.state.status = RunStatus::Blocked;
2894 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2895 self.state.save()?;
2896 bail!(
2897 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2898 );
2899 }
2900 self.after_implement()
2901 }
2902
2903 async fn persist_candidates(&mut self) -> Result<()> {
2905 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2906 for label in labels {
2907 let i = self
2908 .state
2909 .candidates
2910 .iter()
2911 .position(|c| c.label == label)
2912 .context("pending candidate is missing")?;
2913 let c = self.state.candidates[i].clone();
2914 let interrupted = self.state.interrupted_implementations.contains(&label);
2915 let result = async {
2916 let wait = crate::repo_lock::WaitPolicy::new(
2917 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2918 || false,
2919 );
2920 let _coordination =
2921 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2922 if let Err(e) = git::snapshot_uncommitted(
2925 &c.worktree,
2926 &self
2927 .state
2928 .dir()
2929 .join("artifacts")
2930 .join(format!("candidate-{label}-recovery")),
2931 )
2932 .await
2933 {
2934 self.state.event(
2935 "preserve",
2936 format!("candidate {label} snapshot incomplete: {e:#}"),
2937 );
2938 }
2939 let rescued = git::rescue_commit(
2940 &c.worktree,
2941 &format!("magi: candidate {label} (uncommitted work)"),
2942 )
2943 .await?;
2944 self.state.note_withheld("implement", &rescued.withheld);
2945 let base = &self.state.base_commit;
2946 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2947 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2948 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2949 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2950 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2951 if c.empty && commits > 0 && !patch.trim().is_empty()
2952 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2953 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2954 || self.state.tally.is_some())
2955 {
2956 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2960 }
2961 let c = &mut self.state.candidates[i];
2962 c.commits = commits;
2963 c.stat = stat;
2964 c.files = files;
2965 let no_changes = commits == 0 || patch.trim().is_empty();
2966 c.empty = no_changes && !interrupted;
2967 if interrupted && no_changes {
2968 c.failed = None;
2969 c.verified_noop = None;
2970 }
2971 if !c.empty {
2972 c.failed = None;
2973 c.verified_noop = None;
2974 }
2975 self.state.event(
2976 "preserve",
2977 format!("candidate {label}: {files} files, {commits} commits"),
2978 );
2979 Ok::<_, anyhow::Error>(())
2980 }
2981 .await;
2982 match result {
2983 Ok(()) => {
2984 self.state.candidate_preservation.remove(&label);
2985 if !interrupted || self.state.candidates[i].files > 0 {
2986 self.state.interrupted_implementations.remove(&label);
2987 }
2988 }
2989 Err(e) => {
2990 let reason = format!("candidate {label} preservation failed: {e:#}");
2991 self.state.event("preserve", &reason);
2992 self.state.candidate_preservation.insert(label, reason);
2993 }
2994 }
2995 if self.state.candidate_preservation.is_empty()
2996 && self.state.status == RunStatus::Blocked
2997 && self.state.candidates.iter().all(|c| c.failed.is_none())
2998 {
2999 self.state.status = RunStatus::Implementing;
3003 self.state.event(
3004 "preserve",
3005 "candidate persistence recovered; implementation can continue",
3006 );
3007 }
3008 self.state.save()?;
3009 }
3010 if !self.state.candidate_preservation.is_empty() {
3011 self.state.status = RunStatus::Blocked;
3012 self.state.save()?;
3013 bail!(
3014 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3015 );
3016 }
3017 Ok(())
3018 }
3019
3020 async fn resume_undelivered(
3048 &mut self,
3049 results: &mut [(usize, SeatState, AgentOutcome)],
3050 sent: &[SeatJob],
3051 prompts: &Prompts,
3052 run_id: &str,
3053 ) {
3054 for (wi, seat, out) in results.iter_mut() {
3055 let Some(dropped) = (match &*out {
3056 AgentOutcome::Dropped(o) => o.dropped.clone(),
3057 _ => None,
3058 }) else {
3059 continue;
3060 };
3061 let Some(job) = sent.get(*wi) else { continue };
3062 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3064 self.state.event(
3065 "implement",
3066 format!(
3067 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3068 work is in the tree",
3069 seat.key, dropped.output_tokens, dropped.why
3070 ),
3071 );
3072 continue;
3073 }
3074 if !has_context(&job.spec, seat, job.sessions) {
3082 self.state.event(
3083 "implement",
3084 format!(
3085 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3086 is no session left to resume",
3087 seat.key, dropped.output_tokens, dropped.why
3088 ),
3089 );
3090 continue;
3091 }
3092 self.state.event(
3093 "implement",
3094 format!(
3095 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3096 conversation",
3097 seat.key, dropped.output_tokens, dropped.why
3098 ),
3099 );
3100 let mut retry = job.clone();
3101 retry.seat = seat.clone();
3102 retry.prompt = prompt::resume_after_drop(&dropped.why);
3103 retry.timeout = retry_budget(job.timeout, true);
3104 retry.stem = format!("{}-resume", job.stem);
3105 let cache = self.state.config.cache_dir();
3106 let ctx = WaveCtx {
3107 carry_seats: false,
3108 pauses: Some((&self.pause, &self.interrupt)),
3109 run: run_id,
3110 node: "implement",
3111 prompts,
3112 cache: cache.as_deref(),
3113 round: None,
3114 };
3115 let (resumed_seat, resumed) =
3116 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3117 *seat = resumed_seat;
3118 *out = resumed;
3119 }
3120 }
3121
3122 async fn resume_seat_handovers(
3187 &mut self,
3188 results: &mut [(usize, SeatState, AgentOutcome)],
3189 sent: &mut [SeatJob],
3190 prompts: &Prompts,
3191 run_id: &str,
3192 ) -> Result<()> {
3193 let instruction = seeded_instruction(&self.state);
3194 let language = self.state.config.graph.language.clone();
3195 let github_language = self.state.config.graph.github_language.clone();
3196 let brief = self
3197 .state
3198 .advice
3199 .as_ref()
3200 .and_then(|a| a.synthesis.as_deref())
3201 .map(str::to_owned);
3202 let attachments = self.state.attachments.clone();
3203 for (wi, seat, out) in results.iter_mut() {
3204 let others: BTreeSet<String> = sent
3207 .iter()
3208 .enumerate()
3209 .filter(|(j, _)| j != wi)
3210 .map(|(_, j)| j.spec.id.clone())
3211 .collect();
3212 let Some(job) = sent.get_mut(*wi) else {
3213 continue;
3214 };
3215 let start = self
3220 .roles
3221 .implementer_roster
3222 .iter()
3223 .position(|s| s.id == job.spec.id)
3224 .unwrap_or(0);
3225 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3226 let mut fallback_attempt = 0usize;
3227 let mut prev: Option<FailClass> = None;
3228 while let Some(cur) = FailClass::of(&*out) {
3229 if !should_hand_over(prev.as_ref(), &cur) {
3230 break;
3231 }
3232 let Some(next) =
3233 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3234 .cloned()
3235 else {
3236 break;
3237 };
3238 tried.insert(next.id.clone());
3239 fallback_attempt += 1;
3240
3241 let coordination = self.coordinate(&job.cwd).await?;
3242 if let Ok(r) = git::rescue_commit(
3243 &job.cwd,
3244 &format!(
3245 "magi: candidate {} (uncommitted work before {} fallback)",
3246 seat.key,
3247 if cur == FailClass::Quota {
3248 "quota"
3249 } else {
3250 "handover"
3251 }
3252 ),
3253 )
3254 .await
3255 {
3256 self.state.note_withheld("implement", &r.withheld);
3257 }
3258 drop(coordination);
3259
3260 record_handover(
3261 &mut self.state,
3262 "implement",
3263 &seat.key,
3264 &seat.agent,
3265 &next.id,
3266 &cur,
3267 &fail_reason(&*out),
3268 );
3269 prev = Some(cur.clone());
3270
3271 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3272 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3273 job.spec = next.clone();
3281 let mut retry = job.clone();
3282 retry.seat = new_seat;
3283 retry.prompt = prompt::implement_in(
3284 &instruction,
3285 &job.cwd.to_string_lossy(),
3286 &language,
3287 &github_language,
3288 brief.as_deref(),
3289 &attachments,
3290 );
3291 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3292 let cache = self.state.config.cache_dir();
3293 let ctx = WaveCtx {
3294 carry_seats: false,
3295 pauses: Some((&self.pause, &self.interrupt)),
3296 run: run_id,
3297 node: "implement",
3298 prompts,
3299 cache: cache.as_deref(),
3300 round: None,
3301 };
3302 let (fallback_seat, fallback_out) = run_one(
3303 retry,
3304 Arc::clone(&self.sem),
3305 &ctx,
3306 &mut self.state,
3307 fallback_attempt,
3308 )
3309 .await;
3310 *seat = fallback_seat;
3311 *out = fallback_out;
3312 }
3313 }
3314 Ok(())
3315 }
3316
3317 async fn resume_unconfirmed_commands(
3341 &mut self,
3342 results: &mut [(usize, SeatState, AgentOutcome)],
3343 sent: &[SeatJob],
3344 prompts: &Prompts,
3345 run_id: &str,
3346 ) {
3347 for (wi, seat, out) in results.iter_mut() {
3348 let AgentOutcome::Ok(o) = &*out else {
3349 continue;
3350 };
3351 if !has_unconfirmed_command(&o.commands) {
3352 continue;
3353 }
3354 let Some(job) = sent.get(*wi) else { continue };
3355 if !has_context(&job.spec, seat, job.sessions) {
3356 self.state.event(
3357 "implement",
3358 format!(
3359 "{}: the reply named a command whose own CLI never confirmed the exit \
3360 status of, but there is no session left to resume",
3361 seat.key
3362 ),
3363 );
3364 continue;
3365 }
3366 self.state.event(
3367 "implement",
3368 format!(
3369 "{}: the reply named a command whose own CLI never confirmed the exit \
3370 status of; resuming the conversation",
3371 seat.key
3372 ),
3373 );
3374 let mut retry = job.clone();
3375 retry.seat = seat.clone();
3376 retry.prompt = prompt::resume_incomplete(
3377 "a command in your last reply had no confirmed exit status",
3378 );
3379 retry.timeout = retry_budget(job.timeout, true);
3380 retry.stem = format!("{}-confirm", job.stem);
3381 let cache = self.state.config.cache_dir();
3382 let ctx = WaveCtx {
3383 carry_seats: false,
3384 pauses: Some((&self.pause, &self.interrupt)),
3385 run: run_id,
3386 node: "implement",
3387 prompts,
3388 cache: cache.as_deref(),
3389 round: None,
3390 };
3391 let (resumed_seat, resumed) =
3392 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3393 *seat = resumed_seat;
3394 *out = resumed;
3395 }
3396 }
3397
3398 async fn continue_fix_report(
3419 &mut self,
3420 mut seat: SeatState,
3421 parse_err: String,
3422 job: &SeatJob,
3423 prompts: &Prompts,
3424 run_id: &str,
3425 round: usize,
3426 ) -> (
3427 SeatState,
3428 Option<FixReport>,
3429 Option<String>,
3430 ContinuationRecord,
3431 ) {
3432 let mut last_err = parse_err;
3433 let mut cumulative_wait_ms = 0u64;
3434 let mut attempts = 0usize;
3435 loop {
3436 if !has_context(&job.spec, &seat, job.sessions) {
3437 self.state.event(
3438 "fix",
3439 format!(
3440 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3441 session left to resume into"
3442 ),
3443 );
3444 let outcome = if attempts == 0 {
3445 ContinuationOutcome::NoSession
3446 } else {
3447 ContinuationOutcome::Exhausted
3448 };
3449 return (
3450 seat,
3451 None,
3452 Some(format!("unparsable fix report: {last_err}")),
3453 ContinuationRecord {
3454 attempts,
3455 cumulative_wait_ms,
3456 outcome,
3457 },
3458 );
3459 }
3460 if attempts >= MAX_FIX_CONTINUATIONS {
3461 self.state.event(
3462 "fix",
3463 format!(
3464 "round {round}: fixer's reply still had no adoption report after \
3465 {attempts} continuation(s) ({last_err}); giving up"
3466 ),
3467 );
3468 return (
3469 seat,
3470 None,
3471 Some(format!(
3472 "unparsable fix report after {attempts} continuation(s): {last_err}"
3473 )),
3474 ContinuationRecord {
3475 attempts,
3476 cumulative_wait_ms,
3477 outcome: ContinuationOutcome::Exhausted,
3478 },
3479 );
3480 }
3481 attempts += 1;
3482 self.state.event(
3483 "fix",
3484 format!(
3485 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3486 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3487 ),
3488 );
3489 let mut retry = job.clone();
3490 retry.seat = seat.clone();
3491 retry.prompt = prompt::resume_incomplete(&last_err);
3492 retry.timeout = retry_budget(job.timeout, true);
3493 retry.stem = format!("{}-continue{attempts}", job.stem);
3494 let cache = self.state.config.cache_dir();
3495 let ctx = WaveCtx {
3496 carry_seats: false,
3497 pauses: Some((&self.pause, &self.interrupt)),
3498 run: run_id,
3499 node: "fix",
3500 prompts,
3501 cache: cache.as_deref(),
3502 round: Some(round),
3503 };
3504 let (resumed_seat, resumed_out) = run_one(
3505 retry,
3506 Arc::clone(&self.sem),
3507 &ctx,
3508 &mut self.state,
3509 attempts,
3510 )
3511 .await;
3512 seat = resumed_seat;
3513 match resumed_out {
3514 AgentOutcome::Ok(o) => {
3515 cumulative_wait_ms += o.duration_ms;
3516 match verdict::extract_json::<FixReport>(&o.text) {
3517 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3518 self.state.event(
3519 "fix",
3520 format!(
3521 "round {round}: fixer's adoption report recovered after \
3522 {attempts} continuation(s)"
3523 ),
3524 );
3525 return (
3526 seat,
3527 Some(report),
3528 None,
3529 ContinuationRecord {
3530 attempts,
3531 cumulative_wait_ms,
3532 outcome: ContinuationOutcome::Resumed,
3533 },
3534 );
3535 }
3536 Ok(_) => {
3544 last_err = "the reply parsed, but it reported a command whose own CLI \
3545 never confirmed an exit status"
3546 .to_owned();
3547 }
3548 Err(e) => last_err = e.to_string(),
3549 }
3550 }
3551 AgentOutcome::Quota(o) => {
3552 cumulative_wait_ms += o.duration_ms;
3553 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3554 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3555 self.state.quota.push(loss);
3556 self.state.event(
3557 "fix",
3558 format!(
3559 "round {round}: continuation rate limited (quota); not retrying now"
3560 ),
3561 );
3562 return (
3563 seat,
3564 None,
3565 Some("rate limited (quota) while recovering the fix report".to_owned()),
3566 ContinuationRecord {
3567 attempts,
3568 cumulative_wait_ms,
3569 outcome: ContinuationOutcome::QuotaLost,
3570 },
3571 );
3572 }
3573 AgentOutcome::Dropped(o) => {
3574 cumulative_wait_ms += o.duration_ms;
3575 let why = o
3576 .dropped
3577 .as_ref()
3578 .map(|d| d.why.as_str())
3579 .unwrap_or("the CLI ended the stream without delivering its answer");
3580 last_err = format!("the CLI dropped the stream ({why})");
3581 }
3582 AgentOutcome::Failed(e) => last_err = e,
3583 AgentOutcome::ResourceBlocked(e) => {
3586 return (
3587 seat,
3588 None,
3589 Some(format!("the continuation was not started: {e}")),
3590 ContinuationRecord {
3591 attempts: attempts.saturating_sub(1),
3592 cumulative_wait_ms,
3593 outcome: ContinuationOutcome::NoSession,
3594 },
3595 );
3596 }
3597 }
3598 }
3599 }
3600
3601 fn after_implement(&mut self) -> Result<()> {
3602 if !self.state.candidate_preservation.is_empty() {
3603 bail!("candidate preservation incomplete; judging is unsafe");
3604 }
3605 if self.state.leaks.is_empty() {
3607 let cfg = self.state.config.blind.clone();
3608 let mut leaks = Vec::new();
3609 for c in &self.state.candidates {
3610 let Some(patch) =
3611 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3612 else {
3613 continue;
3614 };
3615 leaks.extend(blind::scan(
3616 &format!("candidate {} patch", c.label),
3617 &patch,
3618 &cfg.vendor_tokens,
3619 ));
3620 }
3621 if !leaks.is_empty() {
3622 let summary = leaks
3623 .iter()
3624 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3625 .collect::<Vec<_>>()
3626 .join(", ");
3627 match cfg.on_leak {
3628 LeakPolicy::Fail => {
3629 self.state.status = RunStatus::Failed;
3630 self.state
3631 .event("blind", format!("vendor text in a patch: {summary}"));
3632 self.state.leaks = leaks;
3633 self.state.save()?;
3634 self.settle_questions();
3635 bail!(
3636 "blind.on_leak = \"fail\" and vendor text reached a \
3637 judged patch: {summary}"
3638 );
3639 }
3640 LeakPolicy::Redact => self.state.event(
3641 "blind",
3642 format!("redacting vendor text for judging: {summary}"),
3643 ),
3644 LeakPolicy::Warn => self.state.event(
3645 "blind",
3646 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3647 ),
3648 }
3649 self.state.leaks = leaks;
3650 }
3651 }
3652
3653 if self.state.viable().is_empty() {
3654 if self.state.all_candidates_verified_noop() {
3655 self.state.status = RunStatus::VerifiedNoop;
3666 self.state.save()?;
3667 self.settle_questions();
3668 return Ok(());
3669 }
3670 self.state.status = RunStatus::Failed;
3671 self.state.save()?;
3672 self.settle_questions();
3673 bail!("no candidate produced a change; nothing to judge");
3674 }
3675 self.state.status = RunStatus::Judging;
3676 self.state.save()?;
3677 Ok(())
3678 }
3679
3680 async fn judge(&mut self) -> Result<()> {
3683 let run_id = self.state.id.clone();
3688 let prompts = self.state.config.prompts.clone();
3689 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3690 return Ok(());
3691 }
3692 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3693 if viable.len() == 1 {
3694 self.state.judge_skipped = true;
3701 self.state.event(
3702 "judge",
3703 format!(
3704 "only candidate {} produced a change; judging skipped",
3705 viable[0].label
3706 ),
3707 );
3708 self.state.save()?;
3709 return Ok(());
3710 }
3711 self.state.status = RunStatus::Judging;
3712
3713 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3714 let language = self.state.config.graph.language.clone();
3715 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3716 let sessions = self.state.config.graph.sessions;
3717 let artifacts = agent::artifacts_dir(&self.state.dir());
3718 let root = self.state.worktree_root();
3719 let base_short = short(&self.state.base_commit);
3720
3721 let mut jobs = Vec::new();
3722 let mut orders = Vec::new();
3723 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3724 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3725 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3726 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3727 let seat_key = format!("judge-{}", j + 1);
3728 let seat = self.seat(&seat_key, &spec.id);
3729 jobs.push(SeatJob {
3730 prompt: prompt::judge(
3731 &self.state.instruction,
3732 &views,
3733 self.roles.judges.len(),
3734 &base_short,
3735 &language,
3736 ),
3737 spec,
3738 seat,
3739 cwd: root.join(format!("judge-{}", j + 1)),
3740 timeout,
3741 allow_write: false,
3742 sessions,
3743 artifacts: artifacts.clone(),
3744 stem: format!("judge-{}", j + 1),
3745 handover: None,
3746 });
3747 }
3748
3749 self.state.event(
3750 "judge",
3751 format!(
3752 "{} judges ranking {} candidates blind",
3753 jobs.len(),
3754 viable.len()
3755 ),
3756 );
3757 let labels_for_check = labels.clone();
3758 let mut quota_losses = Vec::new();
3759 let cache = self.state.config.cache_dir();
3760 let ctx = WaveCtx {
3761 carry_seats: false,
3762 pauses: Some((&self.pause, &self.interrupt)),
3763 run: &run_id,
3764 node: "judge",
3765 prompts: &prompts,
3766 cache: cache.as_deref(),
3767 round: None,
3768 };
3769 let results = ask_json_wave::<Ranking>(
3770 jobs,
3771 Arc::clone(&self.sem),
3772 self.state.config.graph.retries,
3773 &self.roles.judge_roster,
3774 &ctx,
3775 &mut quota_losses,
3776 &mut self.state,
3777 &move |r: &Ranking| r.validate(&labels_for_check),
3778 )
3779 .await;
3780 if self.state.resource_block.is_some() {
3781 return self.stop_for_resource();
3782 }
3783 self.state.quota.extend(quota_losses);
3784
3785 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3786 let agent_id = seat.agent.clone();
3787 self.state.seats.insert(seat.key.clone(), seat);
3788 let mut record = Judgement {
3789 judge: j + 1,
3790 seat: format!("judge-{}", j + 1),
3791 agent: agent_id,
3792 ranking: Vec::new(),
3793 reasons: BTreeMap::new(),
3794 confidence: None,
3795 strengths: BTreeMap::new(),
3796 order: orders[j].clone(),
3797 failed: None,
3798 duration_ms: 0,
3799 };
3800 match res {
3801 Ok((ranking, out)) => {
3802 record.ranking = ranking.normalized();
3803 record.reasons = ranking.reasons;
3804 record.strengths = ranking.strengths;
3805 record.confidence = ranking.confidence;
3806 record.duration_ms = out.duration_ms;
3807 self.state.event(
3808 "judge",
3809 format!(
3810 "judge {} ranked {}",
3811 j + 1,
3812 record.ranking.iter().collect::<String>()
3813 ),
3814 );
3815 }
3816 Err(e) => {
3817 record.failed = Some(e.to_string());
3818 self.state
3819 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3820 }
3821 }
3822 self.state.judgements.push(record);
3823 self.state.save()?;
3824 }
3825 Ok(())
3826 }
3827
3828 async fn deliberate(&mut self) -> Result<()> {
3831 if self.hold_base_sync().await? {
3832 return Ok(());
3833 }
3834 let run_id = self.state.id.clone();
3839 let prompts = self.state.config.prompts.clone();
3840 let rounds = self.state.config.graph.deliberate_rounds;
3841 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
3843 return Ok(());
3844 }
3845 let tops: Vec<char> = self
3846 .state
3847 .judgements
3848 .iter()
3849 .filter_map(|j| j.ranking.first().copied())
3850 .collect();
3851 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3852 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3853 self.state.event(
3854 "deliberate",
3855 format!("judges agreed on {} outright; no deliberation", tops[0]),
3856 );
3857 }
3858 self.state.status = RunStatus::Voting;
3859 self.state.save()?;
3860 return Ok(());
3861 }
3862
3863 self.state.status = RunStatus::Deliberating;
3864 self.state.event(
3865 "deliberate",
3866 format!(
3867 "split: first choices were {} — opening {rounds} round(s)",
3868 tops.iter().collect::<String>()
3869 ),
3870 );
3871
3872 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3873 let language = self.state.config.graph.language.clone();
3874 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3875 let sessions = self.state.config.graph.sessions;
3876 let artifacts = agent::artifacts_dir(&self.state.dir());
3877 let root = self.state.worktree_root();
3878 let base_short = short(&self.state.base_commit);
3879
3880 for round in self.state.deliberation.len() + 1..=rounds {
3884 let mut turns: Vec<DeliberationTurn> = Vec::new();
3885 let mut round_losses = Vec::new();
3886 let mut asked = 0;
3887 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3888 if self.state.judgements[j].failed.is_some() {
3889 continue;
3890 }
3891 let seat_key = format!("judge-{}", j + 1);
3892 let spec = self.occupant(&seat_key, spec);
3893 let mut seat = self.seat(&seat_key, &spec.id);
3894 let transcript = self.transcript(&turns, j);
3895 let build = |context: Option<&str>| {
3896 prompt::deliberate(
3897 &self.state.instruction,
3898 context,
3899 &transcript,
3900 round,
3901 rounds,
3902 &language,
3903 )
3904 };
3905 let block = self.candidate_block(&viable, &base_short);
3906 let full = build(Some(&block));
3907 let text = if has_context(&spec, &seat, sessions) {
3908 build(None)
3909 } else {
3910 full.clone()
3911 };
3912 let job = SeatJob {
3913 spec,
3914 seat: seat.clone(),
3915 prompt: text,
3916 cwd: root.join(format!("judge-{}", j + 1)),
3917 timeout,
3918 allow_write: false,
3919 sessions,
3920 artifacts: artifacts.clone(),
3921 stem: format!("delib-{round}-judge-{}", j + 1),
3922 handover: Some(full),
3923 };
3924 let cache = self.state.config.cache_dir();
3925 let ctx = WaveCtx {
3926 carry_seats: false,
3927 pauses: Some((&self.pause, &self.interrupt)),
3928 run: &run_id,
3929 node: "deliberate",
3930 prompts: &prompts,
3931 cache: cache.as_deref(),
3932 round: None,
3933 };
3934 let mut losses = Vec::new();
3938 asked += 1;
3939 let mut results = ask_wave_with::<String>(
3940 vec![job],
3941 Arc::clone(&self.sem),
3942 0,
3943 &self.roles.judge_roster,
3944 &ctx,
3945 &mut losses,
3946 &mut self.state,
3947 &|text: &str| {
3948 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3949 },
3950 )
3951 .await;
3952 if self.state.resource_block.is_some() {
3953 return self.stop_for_resource();
3954 }
3955 round_losses.extend(losses.iter().cloned());
3956 self.state.quota.extend(losses);
3957 let (updated, res, _) = results.pop().expect("one job in, one result out");
3958 seat = updated;
3959 let agent_id = seat.agent.clone();
3960 self.state.seats.insert(seat.key.clone(), seat);
3961 let body = match res {
3962 Ok((body, _)) => body,
3963 Err(e) => {
3966 self.state
3967 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3968 continue;
3969 }
3970 };
3971 let tentative = verdict::extract_json::<Position>(&body)
3972 .ok()
3973 .and_then(|p| p.tentative)
3974 .and_then(|s| s.trim().chars().next())
3975 .map(|c| c.to_ascii_uppercase());
3976 self.state.event(
3977 "deliberate",
3978 format!(
3979 "round {round}: judge {} now favours {}",
3980 j + 1,
3981 tentative.map_or("—".to_owned(), |c| c.to_string())
3982 ),
3983 );
3984 turns.push(DeliberationTurn {
3985 judge: j + 1,
3986 agent: agent_id,
3987 body: blind::sanitize_prose(&body, &self.state.config.blind),
3988 tentative,
3989 });
3990 }
3991 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
3992 block_machine_outage(&mut self.state, "deliberate", &round_losses);
3993 return self.stop_for_resource();
3994 }
3995 self.state
3996 .deliberation
3997 .push(DeliberationRound { round, turns });
3998 self.state.save()?;
3999 }
4000
4001 self.state.status = RunStatus::Voting;
4002 self.state.save()?;
4003 Ok(())
4004 }
4005
4006 async fn vote(&mut self) -> Result<()> {
4009 if self.hold_base_sync().await? {
4010 return Ok(());
4011 }
4012 let run_id = self.state.id.clone();
4017 let prompts = self.state.config.prompts.clone();
4018 if !self.state.votes.is_empty() || self.state.tally.is_some() {
4019 return Ok(());
4020 }
4021 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4022 if viable.len() == 1 {
4023 return Ok(());
4024 }
4025 self.state.status = RunStatus::Voting;
4026
4027 let language = self.state.config.graph.language.clone();
4028 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4029 let sessions = self.state.config.graph.sessions;
4030 let artifacts = agent::artifacts_dir(&self.state.dir());
4031 let root = self.state.worktree_root();
4032 let base_short = short(&self.state.base_commit);
4033 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4034
4035 let mut jobs = Vec::new();
4036 let mut seats_at = Vec::new();
4037 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4038 if self
4039 .state
4040 .judgements
4041 .get(j)
4042 .is_some_and(|r| r.failed.is_some())
4043 {
4044 continue;
4045 }
4046 let seat_key = format!("judge-{}", j + 1);
4047 let spec = self.occupant(&seat_key, spec);
4048 let seat = self.seat(&seat_key, &spec.id);
4049 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4050 let text = if has_context(&spec, &seat, sessions) {
4051 prompt::final_vote(&viable, &language)
4052 } else {
4053 full.clone()
4054 };
4055 jobs.push(SeatJob {
4056 spec,
4057 seat,
4058 prompt: text,
4059 cwd: root.join(format!("judge-{}", j + 1)),
4060 timeout,
4061 allow_write: false,
4062 sessions,
4063 artifacts: artifacts.clone(),
4064 stem: format!("vote-judge-{}", j + 1),
4065 handover: Some(full),
4066 });
4067 seats_at.push(j);
4068 }
4069
4070 self.state.event(
4071 "vote",
4072 format!(
4073 "collecting {} final votes one by one, privately",
4074 jobs.len()
4075 ),
4076 );
4077 let allowed = viable.clone();
4078 let mut quota_losses = Vec::new();
4079 let cache = self.state.config.cache_dir();
4080 let ctx = WaveCtx {
4081 carry_seats: false,
4082 pauses: Some((&self.pause, &self.interrupt)),
4083 run: &run_id,
4084 node: "vote",
4085 prompts: &prompts,
4086 cache: cache.as_deref(),
4087 round: None,
4088 };
4089 let results = ask_json_wave::<FinalVote>(
4090 jobs,
4091 Arc::clone(&self.sem),
4092 self.state.config.graph.retries,
4093 &self.roles.judge_roster,
4094 &ctx,
4095 &mut quota_losses,
4096 &mut self.state,
4097 &move |v: &FinalVote| match v.label() {
4098 Some(c) if allowed.contains(&c) => Ok(()),
4099 other => bail!("vote {other:?} is not one of {allowed:?}"),
4100 },
4101 )
4102 .await;
4103 if self.state.resource_block.is_some() {
4104 return self.stop_for_resource();
4105 }
4106 self.state.quota.extend(quota_losses);
4107
4108 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4109 let agent_id = seat.agent.clone();
4110 self.state.seats.insert(seat.key.clone(), seat);
4111 let initial = self
4112 .state
4113 .judgements
4114 .get(j)
4115 .and_then(|r| r.ranking.first().copied());
4116 let mut record = VoteRecord {
4117 judge: j + 1,
4118 agent: agent_id,
4119 vote: None,
4120 reason: String::new(),
4121 changed: false,
4122 };
4123 match res {
4124 Ok((v, _)) => {
4125 record.vote = v.label();
4126 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4127 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4128 self.state.event(
4129 "vote",
4130 format!(
4131 "judge {} voted {}{}",
4132 j + 1,
4133 record.vote.unwrap_or('?'),
4134 if record.changed { " (changed)" } else { "" }
4135 ),
4136 );
4137 }
4138 Err(e) => {
4139 self.state
4140 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4141 }
4142 }
4143 self.state.votes.push(record);
4144 self.state.save()?;
4145 }
4146 Ok(())
4147 }
4148
4149 fn tally(&mut self) -> Result<()> {
4152 if self.state.tally.is_some() {
4153 return Ok(());
4154 }
4155 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4156 let tops: Vec<char> = self
4157 .state
4158 .judgements
4159 .iter()
4160 .filter_map(|j| j.ranking.first().copied())
4161 .collect();
4162 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4163
4164 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4167 let mut cast: Vec<char> = Vec::new();
4168 for (i, j) in self.state.judgements.iter().enumerate() {
4169 let vote = self
4170 .state
4171 .votes
4172 .iter()
4173 .find(|v| v.judge == i + 1)
4174 .and_then(|v| v.vote)
4175 .or_else(|| j.ranking.first().copied());
4176 if let Some(v) = vote {
4177 *first_choice.entry(v).or_insert(0) += 1;
4178 cast.push(v);
4179 }
4180 }
4181
4182 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4183 for j in &self.state.judgements {
4184 let n = j.ranking.len();
4185 for (pos, label) in j.ranking.iter().enumerate() {
4186 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4187 }
4188 }
4189
4190 let best = first_choice.values().copied().max().unwrap_or(0);
4191 let mut leaders: Vec<char> = first_choice
4192 .iter()
4193 .filter(|(_, v)| **v == best)
4194 .map(|(k, _)| *k)
4195 .collect();
4196 let mut tie_break = None;
4197 if leaders.len() > 1 {
4198 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4199 let borda_leaders: Vec<char> = leaders
4200 .iter()
4201 .copied()
4202 .filter(|l| borda[l] == top_borda)
4203 .collect();
4204 tie_break = Some(if borda_leaders.len() == 1 {
4205 format!(
4206 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4207 leaders.len()
4208 )
4209 } else {
4210 format!(
4211 "{} way tie on both first-choice votes and Borda points, broken by label order",
4212 leaders.len()
4213 )
4214 });
4215 leaders = borda_leaders;
4216 leaders.sort_unstable();
4217 }
4218 let winner = *leaders
4219 .first()
4220 .or(viable.first())
4221 .context("no candidate to declare a winner from")?;
4222
4223 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4224 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4225 let deliberated = !self.state.deliberation.is_empty();
4226
4227 let quota_seats: std::collections::BTreeSet<&str> =
4231 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4232 let mut present = 0usize;
4233 for (i, j) in self.state.judgements.iter().enumerate() {
4234 if quota_seats.contains(j.seat.as_str()) {
4235 continue;
4236 }
4237 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4238 let voted = self
4239 .state
4240 .votes
4241 .iter()
4242 .any(|v| v.judge == i + 1 && v.vote.is_some());
4243 if ranked || voted {
4244 present += 1;
4245 }
4246 }
4247 let needs_quorum = viable.len() > 1;
4253 let judges_total = if needs_quorum {
4254 self.roles.judges.len()
4255 } else {
4256 0
4257 };
4258 let quorum = if needs_quorum {
4259 judges_total / 2 + 1
4260 } else {
4261 0
4262 };
4263 let met_quorum = !needs_quorum || present >= quorum;
4264 let uncontested = (!needs_quorum).then(|| {
4265 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4266 });
4267
4268 self.state.event(
4269 "tally",
4270 match &uncontested {
4271 Some(reason) => format!("winner {winner} — {reason}"),
4272 None => format!(
4273 "winner {winner} — votes {} | initial {} | {} changed | \
4274 {present}/{judges_total} judges{}",
4275 first_choice
4276 .iter()
4277 .map(|(k, v)| format!("{k}:{v}"))
4278 .collect::<Vec<_>>()
4279 .join(" "),
4280 if unanimous_initial {
4281 "unanimous"
4282 } else {
4283 "split"
4284 },
4285 changed_votes,
4286 if met_quorum {
4287 String::new()
4288 } else {
4289 format!(" — below quorum ({quorum} required)")
4290 },
4291 ),
4292 },
4293 );
4294 if !met_quorum {
4295 self.state.event(
4296 "stall",
4297 format!(
4298 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4299 the run stops here, resumable"
4300 ),
4301 );
4302 }
4303 self.state.tally = Some(Tally {
4304 first_choice,
4305 borda,
4306 winner,
4307 rankings: tops.len(),
4308 unanimous_initial,
4309 deliberated,
4310 changed_votes,
4311 unanimous_final,
4312 tie_break,
4313 judges: judges_total,
4314 present,
4315 quorum,
4316 met_quorum,
4317 uncontested,
4318 });
4319 self.state.status = if met_quorum {
4320 RunStatus::Reviewing
4321 } else {
4322 RunStatus::Stalled
4323 };
4324 self.state.save()?;
4325 Ok(())
4326 }
4327
4328 #[allow(clippy::too_many_lines)]
4349 async fn recover_stall(&mut self) -> Result<bool> {
4350 let run_id = self.state.id.clone();
4355 let prompts = self.state.config.prompts.clone();
4356 let quota_seats: BTreeSet<&str> =
4361 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4362 let absent: Vec<String> = self
4363 .state
4364 .judgements
4365 .iter()
4366 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4367 .map(|j| j.seat.clone())
4368 .collect();
4369 if absent.is_empty() {
4370 return Ok(false);
4371 }
4372 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4373 if viable.len() <= 1 {
4374 return Ok(false);
4375 }
4376 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4377 let language = self.state.config.graph.language.clone();
4378 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4379 let sessions = self.state.config.graph.sessions;
4380 let artifacts = agent::artifacts_dir(&self.state.dir());
4381 let root = self.state.worktree_root();
4382 let base_short = short(&self.state.base_commit);
4383 let candidates: Vec<Candidate> = viable.clone();
4384
4385 let mut positions: Vec<usize> = absent
4387 .iter()
4388 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4389 .collect();
4390 if positions.is_empty() {
4391 return Ok(false);
4392 }
4393 positions.sort_unstable();
4394 positions.dedup();
4395
4396 let mut judge_jobs = Vec::new();
4398 for &j in &positions {
4399 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4400 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4401 let seat_key = format!("judge-{}", j + 1);
4402 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4403 let seat = self.seat(&seat_key, &spec.id);
4404 judge_jobs.push(SeatJob {
4405 spec,
4406 seat,
4407 prompt: prompt::judge(
4408 &self.state.instruction,
4409 &views,
4410 self.roles.judges.len(),
4411 &base_short,
4412 &language,
4413 ),
4414 cwd: root.join(seat_key),
4415 timeout,
4416 allow_write: false,
4417 sessions,
4418 artifacts: artifacts.clone(),
4419 stem: format!("judge-{}-recover", j + 1),
4420 handover: None,
4421 });
4422 }
4423
4424 let labels_for_check = labels.clone();
4425 let mut judge_losses = Vec::new();
4426 let retries = self.state.config.graph.retries;
4427 let cache = self.state.config.cache_dir();
4428 let ctx = WaveCtx {
4429 carry_seats: false,
4430 pauses: Some((&self.pause, &self.interrupt)),
4431 run: &run_id,
4432 node: "judge",
4433 prompts: &prompts,
4434 cache: cache.as_deref(),
4435 round: None,
4436 };
4437 let results = ask_json_wave::<Ranking>(
4438 judge_jobs,
4439 Arc::clone(&self.sem),
4440 retries,
4441 &self.roles.judge_roster,
4442 &ctx,
4443 &mut judge_losses,
4444 &mut self.state,
4445 &move |r: &Ranking| r.validate(&labels_for_check),
4446 )
4447 .await;
4448 if self.state.resource_block.is_some() {
4449 self.stop_for_resource()?;
4450 return Ok(false);
4451 }
4452
4453 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4455 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4456 let agent_id = seat.agent.clone();
4457 self.state.seats.insert(seat.key.clone(), seat);
4458 let record = &mut self.state.judgements[j];
4459 match res {
4460 Ok((ranking, out)) => {
4461 record.agent = agent_id;
4462 record.ranking = ranking.normalized();
4463 record.reasons = ranking.reasons;
4464 record.strengths = ranking.strengths;
4465 record.confidence = ranking.confidence;
4466 record.failed = None;
4467 record.duration_ms = out.duration_ms;
4468 recovered.insert(j);
4469 self.state.event(
4470 "recover",
4471 format!("judge {} ranked again after the limit", j + 1),
4472 );
4473 }
4474 Err(e) => {
4475 self.state
4476 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4477 }
4478 }
4479 }
4480
4481 let mut vote_jobs = Vec::new();
4483 let mut vote_pos: Vec<usize> = Vec::new();
4484 for &j in &recovered {
4485 let seat_key = format!("judge-{}", j + 1);
4486 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4487 let seat = self.seat(&seat_key, &spec.id);
4488 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4489 let text = if has_context(&spec, &seat, sessions) {
4490 prompt::final_vote(&labels, &language)
4491 } else {
4492 full.clone()
4493 };
4494 vote_jobs.push(SeatJob {
4495 spec,
4496 seat,
4497 prompt: text,
4498 cwd: root.join(seat_key),
4499 timeout,
4500 allow_write: false,
4501 sessions,
4502 artifacts: artifacts.clone(),
4503 stem: format!("vote-judge-{}-recover", j + 1),
4504 handover: Some(full),
4505 });
4506 vote_pos.push(j);
4507 }
4508 let allowed = labels.clone();
4509 let mut vote_losses = Vec::new();
4510 let vote_retries = self.state.config.graph.retries;
4511 let vote_cache = self.state.config.cache_dir();
4512 let ctx = WaveCtx {
4513 carry_seats: false,
4514 pauses: Some((&self.pause, &self.interrupt)),
4515 run: &run_id,
4516 node: "vote",
4517 prompts: &prompts,
4518 cache: vote_cache.as_deref(),
4519 round: None,
4520 };
4521 let votes = ask_json_wave::<FinalVote>(
4522 vote_jobs,
4523 Arc::clone(&self.sem),
4524 vote_retries,
4525 &self.roles.judge_roster,
4526 &ctx,
4527 &mut vote_losses,
4528 &mut self.state,
4529 &move |v: &FinalVote| match v.label() {
4530 Some(c) if allowed.contains(&c) => Ok(()),
4531 other => bail!("vote {other:?} is not one of {allowed:?}"),
4532 },
4533 )
4534 .await;
4535 if self.state.resource_block.is_some() {
4536 self.stop_for_resource()?;
4537 return Ok(false);
4538 }
4539 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4540 let agent_id = seat.agent.clone();
4541 self.state.seats.insert(seat.key.clone(), seat);
4542 match res {
4543 Ok((v, _)) => {
4544 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4545 rec.vote = v.label();
4546 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4547 } else {
4548 self.state.votes.push(VoteRecord {
4549 judge: j + 1,
4550 agent: agent_id,
4551 vote: v.label(),
4552 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4553 changed: false,
4554 });
4555 }
4556 self.state.event(
4557 "recover",
4558 format!("judge {} voted again after the limit", j + 1),
4559 );
4560 }
4561 Err(e) => {
4562 self.state
4563 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4564 }
4565 }
4566 }
4567
4568 let recovered_keys: BTreeSet<String> = recovered
4572 .iter()
4573 .map(|&j| format!("judge-{}", j + 1))
4574 .collect();
4575 self.state
4576 .quota
4577 .retain(|q| !recovered_keys.contains(&q.seat));
4578 for loss in judge_losses.into_iter().chain(vote_losses) {
4582 if recovered_keys.contains(&loss.seat) {
4583 continue;
4584 }
4585 self.state.quota.retain(|q| q.seat != loss.seat);
4586 self.state.quota.push(loss);
4587 }
4588
4589 self.state.tally = None;
4591 self.tally()?;
4592 Ok(self
4593 .state
4594 .tally
4595 .as_ref()
4596 .map(|t| t.met_quorum)
4597 .unwrap_or(false))
4598 }
4599
4600 async fn graft(&mut self) -> Result<()> {
4616 let Err(e) = self.graft_inner().await else {
4617 return Ok(());
4618 };
4619 if e.is::<crate::repo_lock::ResourceBlocked>() {
4620 return Err(e);
4621 }
4622 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4623 self.state.status = RunStatus::Blocked;
4624 self.state.event("graft", format!("blocked: {e:#}"));
4625 self.state.save()?;
4626 return Err(e);
4627 }
4628 if let Some(rec) = self.state.graft.clone()
4633 && !rec.outcome.settled()
4634 {
4635 if let Some(winner) = self.state.winner().cloned() {
4636 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4637 if re.is::<crate::repo_lock::ResourceBlocked>() {
4638 return Err(re);
4639 }
4640 self.state.status = RunStatus::Blocked;
4641 self.state.event("graft", format!("blocked: {re:#}"));
4642 self.state.save()?;
4643 return Err(re);
4644 }
4645 }
4646 if let Some(g) = self.state.graft.as_mut() {
4647 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4648 }
4649 }
4650 self.state.event("graft", format!("skipped: {e:#}"));
4651 if let Err(e) = self.state.save() {
4652 tracing::warn!("save after graft: {e:#}");
4653 }
4654 Ok(())
4655 }
4656
4657 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4660 let _coordination = self.coordinate(wt).await?;
4661 crate::recovery::restore_repair(wt, pre)
4662 .await
4663 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4664 }
4665
4666 fn graft_skipped(&mut self, why: &str) {
4668 let message = format!("skipped: {why}");
4669 if !self
4670 .state
4671 .events
4672 .iter()
4673 .any(|e| e.node == "graft" && e.message == message)
4674 {
4675 self.state.event("graft", message);
4676 }
4677 }
4678
4679 async fn graft_inner(&mut self) -> Result<()> {
4680 if let Some(rec) = self.state.graft.clone()
4683 && !rec.outcome.settled()
4684 {
4685 if self.state.reviews.is_empty()
4689 && let Some(winner) = self.state.winner().cloned()
4690 {
4691 self.graft_restore(&winner.worktree, &rec.pre).await?;
4692 }
4693 if rec.outcome == GraftOutcome::WaitingForRestore {
4694 self.state.graft = None;
4695 self.state.save()?;
4696 } else {
4697 if let Some(g) = self.state.graft.as_mut() {
4698 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4699 }
4700 self.state.event(
4701 "graft",
4702 "rolled back: the previous round was interrupted; the winner is untouched",
4703 );
4704 self.state.save()?;
4705 return Ok(());
4706 }
4707 }
4708 let plan = match graft::plan(&self.state) {
4709 Ok(plan) => plan,
4710 Err(skip) => {
4711 if skip != graft::Skip::AlreadyDone {
4712 self.graft_skipped(skip.reason());
4713 self.state.save()?;
4714 }
4715 return Ok(());
4716 }
4717 };
4718 let Some(winner) = self.state.winner().cloned() else {
4719 return Ok(());
4720 };
4721 let wt = winner.worktree.clone();
4722 if !git::is_clean(&wt).await? {
4723 self.graft_skipped("the winner's worktree has uncommitted changes");
4724 self.state.save()?;
4725 return Ok(());
4726 }
4727 let pre = git::rev_parse(&wt, "HEAD").await?;
4728 self.state.graft = Some(GraftRecord {
4731 pre: pre.clone(),
4732 post: None,
4733 lines: None,
4734 limit: None,
4735 agent: None,
4736 outcome: GraftOutcome::Running,
4737 requested: graft::requested(&plan),
4738 adoption: None,
4739 adoption_error: None,
4740 });
4741 self.state.event(
4742 "graft",
4743 format!(
4744 "porting strengths of candidate(s) {} onto {}",
4745 plan.strengths
4746 .iter()
4747 .map(|(l, _, _)| l.to_string())
4748 .collect::<Vec<_>>()
4749 .join(", "),
4750 plan.winner
4751 ),
4752 );
4753 self.state.save()?;
4754
4755 let fix_prompt = prompt::graft(
4756 &self.state.instruction,
4757 plan.winner,
4758 &plan.strengths,
4759 &self.state.config.graph.language,
4760 );
4761 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4762 let sessions = self.state.config.graph.sessions;
4763 let artifacts = agent::artifacts_dir(&self.state.dir());
4764 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4765 let (_, seat, out) = self
4766 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4767 SeatJob {
4768 prompt: fix_prompt.clone(),
4769 spec,
4770 seat,
4771 cwd: wt.clone(),
4772 timeout,
4773 allow_write: true,
4774 sessions,
4775 artifacts: artifacts.clone(),
4776 stem: "graft".to_owned(),
4777 handover: None,
4778 }
4779 })
4780 .await;
4781 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4782 self.state.graft.as_mut().expect("graft record").outcome =
4785 GraftOutcome::WaitingForRestore;
4786 return self.stop_for_resource();
4787 }
4788 let mut why: Option<String> = None;
4789 match &out {
4790 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4791 why = Some("the fixer produced nothing usable".to_owned());
4792 }
4793 AgentOutcome::Ok(_) => {}
4794 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4795 AgentOutcome::Quota(o) => {
4796 self.state.quota.push(QuotaLoss::from_report(
4797 seat.key.clone(),
4798 "graft",
4799 o.quota.as_ref(),
4800 ));
4801 why = Some("rate limited (quota)".to_owned());
4802 }
4803 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4804 why = Some(format!(
4805 "the fixer failed: {}",
4806 e.lines()
4807 .next()
4808 .unwrap_or("")
4809 .chars()
4810 .take(160)
4811 .collect::<String>()
4812 ));
4813 }
4814 }
4815 let agent_id = seat.agent.clone();
4816 self.state.seats.insert(seat.key.clone(), seat);
4817
4818 let mut post = None;
4819 let mut lines = None;
4820 let _coordination = self.coordinate(&wt).await?;
4821 if why.is_none() {
4822 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4823 if let Ok(r) = &rescued {
4824 self.state.note_withheld("graft", &r.withheld);
4825 }
4826 let head = git::rev_parse(&wt, "HEAD").await?;
4827 let dirty = match &rescued {
4831 Ok(r) => git::status_porcelain(&wt)
4832 .await
4833 .map(|st| {
4834 st.lines().any(|l| {
4835 !r.withheld
4836 .iter()
4837 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4838 })
4839 })
4840 .unwrap_or(true),
4841 Err(_) => true,
4842 };
4843 if dirty {
4844 why = Some("the fixer left changes that could not be committed".to_owned());
4845 } else if head == pre {
4846 why = Some("the fixer changed nothing".to_owned());
4847 } else {
4848 lines = git::changed_lines(&wt, &pre, &head).await;
4849 post = Some(head);
4850 }
4851 }
4852 let (adoption, adoption_error) = match &out {
4853 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4854 _ => (None, None),
4855 };
4856 let requested = graft::requested(&plan);
4857 let outcome = match why {
4858 None => {
4859 self.state.event(
4860 "graft",
4861 format!(
4862 "applied: {}",
4863 lines.map_or_else(
4864 || "changed lines unavailable".to_owned(),
4865 |n| format!("{n} changed lines")
4866 )
4867 ),
4868 );
4869 GraftOutcome::Applied
4870 }
4871 Some(why) => {
4872 crate::recovery::restore_repair(&wt, &pre)
4875 .await
4876 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4877 self.state.event(
4878 "graft",
4879 format!("rolled back: {why}; the winner is untouched"),
4880 );
4881 GraftOutcome::RolledBack(why)
4882 }
4883 };
4884 self.state.graft = Some(GraftRecord {
4885 pre,
4886 post,
4887 lines,
4888 limit: None,
4889 agent: Some(agent_id),
4890 outcome,
4891 requested,
4892 adoption,
4893 adoption_error,
4894 });
4895 self.state.save()?;
4896 Ok(())
4897 }
4898
4899 async fn fold_losers(&mut self) -> Result<()> {
4902 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4903 return Ok(());
4904 };
4905 if !self.state.candidate_preservation.is_empty() {
4906 bail!("candidate preservation incomplete; refusing cleanup");
4907 }
4908 let repo = self.state.repo.clone();
4909 let _coordination = self.coordinate(&repo).await?;
4910 let mut folded = Vec::new();
4911 for i in 0..self.state.candidates.len() {
4912 let c = &self.state.candidates[i];
4913 if c.label == winner || c.folded {
4914 continue;
4915 }
4916 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4917 remove_preserved_worktree(&repo, &wt).await?;
4918 if git::branch_exists(&repo, &branch).await?
4919 && !git::branch_delete(&repo, &branch).await?
4920 {
4921 bail!("candidate branch removal refused");
4922 }
4923 self.state.candidates[i].folded = true;
4924 self.state.save()?;
4925 folded.push(label.to_string());
4926 }
4927 let root = self.state.worktree_root();
4929 for j in 1..=self.roles.judges.len() {
4930 let wt = root.join(format!("judge-{j}"));
4931 if wt.exists() {
4932 remove_preserved_worktree(&repo, &wt).await?;
4933 }
4934 }
4935 if self.state.config.graph.advise {
4938 for k in 1..=self.state.config.graph.advisors {
4939 let wt = root.join(format!("advisor-{k}"));
4940 if wt.exists() {
4941 remove_preserved_worktree(&repo, &wt).await?;
4942 }
4943 }
4944 }
4945 if !folded.is_empty() {
4946 self.state
4947 .event("fold", format!("folded candidates {}", folded.join(", ")));
4948 self.state.save()?;
4949 }
4950 Ok(())
4951 }
4952
4953 async fn hold_base_sync(&mut self) -> Result<bool> {
4956 self.hold_base_sync_as(None).await
4957 }
4958
4959 async fn hold_base_sync_kind(
4960 &mut self,
4961 kind: crate::base_sync_recovery::StopKind,
4962 ) -> Result<()> {
4963 self.hold_base_sync_as(Some(kind)).await?;
4964 Ok(())
4965 }
4966
4967 async fn hold_base_sync_as(
4968 &mut self,
4969 kind: Option<crate::base_sync_recovery::StopKind>,
4970 ) -> Result<bool> {
4971 if !self.state.base_sync_blocked() {
4972 self.state.base_sync_recovery.stop = None;
4973 return Ok(false);
4974 }
4975 let head = match self.state.winner() {
4976 Some(w) => git::rev_parse(&w.worktree, "HEAD")
4977 .await
4978 .unwrap_or_default(),
4979 None => String::new(),
4980 };
4981 self.state.record_base_sync_stop(&head);
4982 if let Some(kind) = kind {
4983 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4984 }
4985 self.state.save()?;
4986 Ok(true)
4987 }
4988
4989 async fn sync_to_base(&mut self) -> Result<()> {
5031 if self.state.status == RunStatus::AlreadyInBase {
5032 return Ok(());
5033 }
5034 self.hold_base_sync().await?;
5035 let conflicted = self
5036 .state
5037 .base_sync
5038 .as_ref()
5039 .is_some_and(|s| s.conflict.is_some());
5040 let Some(winner) = self.state.winner().cloned() else {
5041 return Ok(());
5042 };
5043
5044 let repo = self.state.repo.clone();
5045 let remote = self.state.config.merge.remote.clone();
5046 let base_branch = self.state.base_branch.clone();
5047 let tracking = format!("{remote}/{base_branch}");
5048
5049 let fetched_base =
5050 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5051 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
5056 return Ok(());
5057 };
5058
5059 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5060 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5061 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5062
5063 if (behind > 0 || conflicted || head == tip)
5072 && self
5073 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5074 .await?
5075 {
5076 return Ok(());
5077 }
5078 if conflicted && self.state.base_recovery_pending() {
5079 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5080 let owner =
5081 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5082 if !fetched_base
5083 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5084 || !git::is_clean(&winner.worktree).await?
5085 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5086 {
5087 return Ok(());
5088 }
5089 }
5090 let recovery = conflicted && self.state.consume_base_recovery(&head);
5091 if conflicted {
5092 self.state.save()?;
5094 if !recovery {
5095 return Ok(());
5096 }
5097 }
5098
5099 if behind == 0 {
5100 if let Some(from) = self
5109 .state
5110 .rebase_fixes
5111 .iter()
5112 .rev()
5113 .find_map(|r| r.from.clone())
5114 && from != head
5115 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5116 .await
5117 .is_ok_and(|o| o.ok())
5118 {
5119 git::sync_to_head(&winner.worktree).await?;
5120 }
5121 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5127 if let Some(why) = &conflict {
5128 self.state.status = RunStatus::Blocked;
5129 self.state.event("land", why.clone());
5130 }
5131 self.state.base_sync = Some(BaseSync {
5132 tip,
5133 behind: 0,
5134 attempts: attempts + usize::from(recovery),
5135 conflict,
5136 already_in: None,
5137 });
5138 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5139 .await?;
5140 self.state.save()?;
5141 return Ok(());
5142 }
5143
5144 if attempts >= BASE_SYNC_ROUNDS && !recovery {
5145 let why = format!(
5146 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5147 synchronization(s); synchronizing again would only race it",
5148 winner.branch
5149 );
5150 self.state.status = RunStatus::Blocked;
5151 self.state.base_sync = Some(BaseSync {
5152 tip,
5153 behind,
5154 attempts,
5155 conflict: Some(why.clone()),
5156 already_in: None,
5157 });
5158 self.state.event("land", why);
5159 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5160 .await?;
5161 self.state.save()?;
5162 return Ok(());
5163 }
5164
5165 if self.state.foreign_checkout.is_some() {
5166 let wait = repository_wait(&self.state, None);
5169 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5170 let merged = git::git_raw(
5171 &winner.worktree,
5172 &[
5173 "-c",
5174 "core.hooksPath=/dev/null",
5175 "merge",
5176 "--no-ff",
5177 "--no-autostash",
5178 "--no-edit",
5179 "-m",
5180 &format!("Merge {tracking} before verifying"),
5181 &tip,
5182 ],
5183 )
5184 .await?;
5185 let conflict = if merged.ok() {
5186 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5187 self.state.event(
5188 "land",
5189 format!("merged {tracking} into {} before verifying", winner.branch),
5190 );
5191 None
5192 } else {
5193 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5194 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5195 }
5196 let why = format!(
5197 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5198 winner.branch,
5199 merged.stderr.chars().take(600).collect::<String>()
5200 );
5201 self.state.status = RunStatus::Blocked;
5202 self.state.event("land", &why);
5203 Some(why)
5204 };
5205 let synced = conflict.is_none();
5206 self.state.base_sync = Some(BaseSync {
5207 tip,
5208 behind: if synced { 0 } else { behind },
5209 attempts: attempts + 1,
5210 conflict,
5211 already_in: None,
5212 });
5213 self.state.save()?;
5214 return Ok(());
5215 }
5216
5217 self.state.event(
5218 "land",
5219 format!(
5220 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5221 winner.branch
5222 ),
5223 );
5224 self.state.save()?;
5225
5226 let branch_tracking = format!("{remote}/{}", winner.branch);
5232 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5233 let remote_tip = if self.state.task_pr_branch().is_none()
5234 && matches!(&fetched_branch, Ok(o) if o.ok())
5235 {
5236 git::rev_parse(&repo, &branch_tracking).await.ok()
5237 } else {
5238 None
5239 };
5240 if let Some(theirs) = &remote_tip
5244 && !git::is_ancestor(&repo, theirs, &head).await
5245 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5246 .await
5247 .is_ok_and(|missing| missing.is_empty())
5248 {
5249 let why = format!(
5250 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5251 them",
5252 short(theirs),
5253 winner.branch
5254 );
5255 self.state.status = RunStatus::Blocked;
5256 self.state.base_sync = Some(BaseSync {
5257 tip,
5258 behind,
5259 attempts: attempts + usize::from(recovery),
5260 conflict: Some(why.clone()),
5261 already_in: None,
5262 });
5263 self.state.event("land", why);
5264 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5265 .await?;
5266 self.state.save()?;
5267 return Ok(());
5268 }
5269
5270 let scratch = self.state.dir().join("base-sync");
5271 let rebased =
5272 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5273 .await
5274 {
5275 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5276 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5277 Err(e) => Err(e),
5278 };
5279 let attempts = attempts + 1;
5280 let stop_kind = match &rebased {
5281 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5282 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5283 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5284 };
5285 match rebased {
5286 Ok(None) => {
5287 git::sync_to_head(&winner.worktree).await?;
5291 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5292 let mut conflict = None;
5293 if let Some(pinned) = &remote_tip {
5294 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5295 match pushed {
5296 Ok(o) if o.ok() => self.state.event(
5297 "land",
5298 format!("pushed rebased {} to {remote}", winner.branch),
5299 ),
5300 Ok(o) => {
5301 conflict = Some(format!(
5302 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5303 winner.branch,
5304 short(pinned),
5305 o.stderr.chars().take(600).collect::<String>()
5306 ));
5307 }
5308 Err(e) => {
5309 conflict = Some(format!(
5310 "rebased {} locally but could not push it: {e:#}",
5311 winner.branch
5312 ));
5313 }
5314 }
5315 }
5316 if let Some(why) = &conflict {
5317 self.state.status = RunStatus::Blocked;
5318 self.state.event("land", why.clone());
5319 }
5320 self.state.base_sync = Some(BaseSync {
5321 tip: tip.clone(),
5322 behind: 0,
5323 attempts,
5324 conflict,
5325 already_in: None,
5326 });
5327 self.state
5328 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5329 }
5330 Ok(Some(conflict)) => {
5331 let why = format!(
5332 "{} conflicts with {tracking} and did not rebase: {}",
5333 winner.branch,
5334 conflict.chars().take(600).collect::<String>()
5335 );
5336 self.state.status = RunStatus::Blocked;
5337 self.state.base_sync = Some(BaseSync {
5338 tip,
5339 behind,
5340 attempts,
5341 conflict: Some(why.clone()),
5342 already_in: None,
5343 });
5344 self.state.event("land", why);
5345 }
5346 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5350 self.state.save()?;
5351 return Ok(());
5352 }
5353 Err(e) => {
5354 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5355 self.state.status = RunStatus::Blocked;
5356 self.state.base_sync = Some(BaseSync {
5357 tip,
5358 behind,
5359 attempts,
5360 conflict: Some(why.clone()),
5361 already_in: None,
5362 });
5363 self.state.event("land", why);
5364 }
5365 }
5366 if !self.state.base_sync_blocked() {
5367 self.state.base_sync_recovery.stop = None;
5368 }
5369 self.hold_base_sync_kind(stop_kind).await?;
5370 self.state.save()?;
5371 Ok(())
5372 }
5373
5374 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5380 if self.state.task_pr_branch().is_some() {
5383 return None;
5384 }
5385 let repo = self.state.repo.clone();
5386 let remote = self.state.config.merge.remote.clone();
5387 let fetched = git::fetch(&repo, &remote, branch).await;
5388 if !matches!(&fetched, Ok(o) if o.ok()) {
5389 return None;
5390 }
5391 let branch_tracking = format!("{remote}/{branch}");
5392 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5393 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5394 return None;
5395 }
5396 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5397 .await
5398 .is_ok_and(|missing| missing.is_empty())
5399 {
5400 return Some(format!(
5401 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5402 them",
5403 short(&theirs)
5404 ));
5405 }
5406 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5407 Ok(o) if o.ok() => {
5408 self.state
5409 .event("land", format!("pushed rebased {branch} to {remote}"));
5410 None
5411 }
5412 Ok(o) => Some(format!(
5413 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5414 someone may have pushed): {}",
5415 short(&theirs),
5416 o.stderr.chars().take(600).collect::<String>()
5417 )),
5418 Err(e) => Some(format!(
5419 "{branch} is rebased locally but could not be pushed: {e:#}"
5420 )),
5421 }
5422 }
5423
5424 async fn settle_already_in(
5441 &mut self,
5442 branch: &str,
5443 tip: &str,
5444 head: &str,
5445 attempts: usize,
5446 behind: usize,
5447 ) -> Result<bool> {
5448 let repo = self.state.repo.clone();
5449 let remote = self.state.config.merge.remote.clone();
5450 let start = self.state.base_commit.clone();
5451 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5452 Ok(Some(e)) => e,
5453 Ok(None) => return Ok(false),
5454 Err(e) => {
5455 tracing::warn!("already-in-base check for {branch}: {e:#}");
5456 return Ok(false);
5457 }
5458 };
5459 let mut verified = vec![head.to_owned()];
5460 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5461 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5462 if matches!(&fetched, Ok(o) if o.ok())
5463 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5464 && theirs != head
5465 {
5466 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5467 Ok(Some(_)) => verified.push(theirs),
5468 _ => return Ok(false),
5469 }
5470 }
5471 let base_branch = self.state.base_branch.clone();
5472 let message = format!(
5473 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5474 land",
5475 evidence.names(),
5476 evidence.proof.as_str()
5477 );
5478 let closed =
5479 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5480 .await;
5481 match closed {
5482 Ok(Ok(url)) => self
5483 .state
5484 .event("land", format!("closed {url}: superseded on {base_branch}")),
5485 Ok(Err(why)) => self
5486 .state
5487 .event("land", format!("did not close a pull request: {why}")),
5488 Err(e) => {
5489 let why = format!(
5490 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5491 not be closed ({e:#}); resume to retry"
5492 );
5493 self.state.status = RunStatus::Blocked;
5494 self.state.base_sync = Some(BaseSync {
5495 tip: tip.to_owned(),
5496 behind,
5497 attempts,
5498 conflict: Some(why.clone()),
5499 already_in: None,
5500 });
5501 self.state.event("land", why);
5502 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5503 .await?;
5504 self.state.save()?;
5505 return Ok(true);
5506 }
5507 }
5508 self.state.status = RunStatus::AlreadyInBase;
5509 self.state.base_sync_recovery.stop = None;
5510 self.state.base_sync = Some(BaseSync {
5511 tip: tip.to_owned(),
5512 behind,
5513 attempts,
5514 conflict: None,
5515 already_in: Some(evidence),
5516 });
5517 self.state.event("land", message);
5518 self.state.save()?;
5519 self.settle_questions();
5520 Ok(true)
5521 }
5522
5523 fn landing_base(&self) -> String {
5533 self.state
5534 .base_sync
5535 .as_ref()
5536 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5537 }
5538
5539 pub async fn fix_selected(
5572 &mut self,
5573 ids: &[String],
5574 reason: &str,
5575 allow_stale: bool,
5576 ) -> Result<()> {
5577 let reason = reason.trim();
5578 if reason.is_empty() {
5579 bail!("a fix request needs a reason — that is the operator's own record of why");
5580 }
5581 if ids.is_empty() {
5582 bail!("no finding id given");
5583 }
5584 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5585 bail!(
5586 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5587 has already concluded — can be given a targeted fix. A run still \
5588 in progress should simply be resumed; a `merged` run's branch has \
5589 already landed, so its answer is a fresh `magi review <branch>`, \
5590 not reopening this run's own record",
5591 self.state.id,
5592 self.state.status.as_str()
5593 );
5594 }
5595 let Some(winner) = self.state.winner().cloned() else {
5596 bail!("run {} has no winning candidate to fix", self.state.id);
5597 };
5598 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5599 bail!(
5600 "branch `{}` no longer exists; this run cannot be extended",
5601 winner.branch
5602 );
5603 }
5604 let home = crate::run::home();
5605 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5606 bail!(
5607 "run {} is currently being worked on by another magi process",
5608 self.state.id
5609 );
5610 }
5611 let _claim = FixClaim::acquire(&self.state.dir())?;
5617
5618 let mut seen = BTreeSet::new();
5622 let mut findings = Vec::new();
5623 let mut missing = Vec::new();
5624 for id in ids {
5625 if !seen.insert(id.clone()) {
5626 continue;
5627 }
5628 match self.state.finding(id) {
5629 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5630 id: f.id.clone(),
5631 severity: f.severity,
5632 reviewer_vote: rec.vote,
5633 round: round.round,
5634 round_head: round.head.clone(),
5635 reviewer: rec.reviewer,
5636 agent: rec.agent.clone(),
5637 file: f.file.clone(),
5638 line: f.line,
5639 title: f.title.clone(),
5640 detail: f.detail.clone(),
5641 outcome: OperatorFixOutcome::Pending,
5642 }),
5643 None => missing.push(id.clone()),
5644 }
5645 }
5646 if !missing.is_empty() {
5647 bail!(
5648 "unknown finding id(s): {}; nothing was changed",
5649 missing.join(", ")
5650 );
5651 }
5652
5653 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5654 let stale_details: Vec<(String, String)> = findings
5655 .iter()
5656 .filter(|f| f.round_head != head_at_request)
5657 .map(|f| (f.id.clone(), f.round_head.clone()))
5658 .collect();
5659 let stale = !stale_details.is_empty();
5660 if stale && !allow_stale {
5661 bail!(
5662 "the branch has moved since some finding(s) were raised — {} — now \
5663 at {}; pass --allow-stale to fix anyway, or re-run review first",
5664 stale_details
5665 .iter()
5666 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5667 .collect::<Vec<_>>()
5668 .join(", "),
5669 short(&head_at_request)
5670 );
5671 }
5672
5673 let request = OperatorFixRequest {
5674 requested_at: Timestamp::now(),
5675 reason: reason.to_owned(),
5676 findings,
5677 head_at_request: head_at_request.clone(),
5678 allow_stale,
5679 stale,
5680 fix: None,
5681 result_head: None,
5682 follow_up_review_run: None,
5683 };
5684 self.state.event(
5685 "fix",
5686 format!(
5687 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5688 request.findings.len(),
5689 request
5690 .findings
5691 .iter()
5692 .map(|f| f.id.as_str())
5693 .collect::<Vec<_>>()
5694 .join(", "),
5695 ),
5696 );
5697 self.state.operator_fixes.push(request);
5704 self.state.save()?;
5705 let request_index = self.state.operator_fixes.len() - 1;
5706
5707 let repo = self.state.repo.clone();
5716 let coordination = self.coordinate(&repo).await?;
5717 if winner.worktree.exists() {
5718 let dirty = git::git(
5721 &winner.worktree,
5722 &["status", "--porcelain", "--untracked-files=all"],
5723 )
5724 .await?;
5725 let only_withheld = dirty.lines().all(|l| {
5726 l.strip_prefix("?? ")
5727 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5728 });
5729 if !only_withheld {
5730 bail!(
5731 "`{}` has uncommitted changes; refusing to touch it — commit or \
5732 discard them first",
5733 winner.worktree.display()
5734 );
5735 }
5736 git::worktree_remove(&self.state.repo, &winner.worktree)
5737 .await
5738 .ok();
5739 }
5740 let fix_worktree = self.state.worktree_root().join("operator-fix");
5741 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5742 git::git(
5743 &self.state.repo,
5744 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5745 )
5746 .await
5747 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5748 if !git::is_clean(&fix_worktree).await? {
5749 git::worktree_remove(&self.state.repo, &fix_worktree)
5750 .await
5751 .ok();
5752 bail!(
5753 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5754 winner.branch
5755 );
5756 }
5757 let repo_for_setup = self.state.repo.clone();
5758 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5759 Ok(()) => {
5760 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5761 }
5762 Err(e) => Err(e),
5763 };
5764 if let Err(e) = setup {
5765 git::worktree_remove(&self.state.repo, &fix_worktree)
5766 .await
5767 .ok();
5768 return Err(e);
5769 }
5770
5771 self.state.resource_block = None;
5774
5775 let run_id = self.state.id.clone();
5776 let prompts = self.state.config.prompts.clone();
5777 let language = self.state.config.graph.language.clone();
5778 let sessions = self.state.config.graph.sessions;
5779 let artifacts = agent::artifacts_dir(&self.state.dir());
5780 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5781 .findings
5782 .iter()
5783 .map(|f| Finding {
5784 id: f.id.clone(),
5785 severity: f.severity,
5786 file: f.file.clone(),
5787 line: f.line,
5788 title: f.title.clone(),
5789 detail: f.detail.clone(),
5790 })
5791 .collect();
5792 let fix_prompt = prompt::operator_fix_in(
5793 &self.state.instruction,
5794 &finding_list,
5795 reason,
5796 &stale_details,
5797 &head_at_request,
5798 &language,
5799 &self.state.config.graph.github_language,
5800 );
5801 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5802 drop(coordination);
5803 let (job, seat, out) = self
5804 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5805 prompt: fix_prompt.clone(),
5806 spec,
5807 seat,
5808 cwd: fix_worktree.clone(),
5809 timeout,
5810 allow_write: true,
5811 sessions,
5812 artifacts: artifacts.clone(),
5813 stem: "operator-fix".to_owned(),
5814 handover: None,
5815 })
5816 .await;
5817 let agent_id = seat.agent.clone();
5818
5819 let mut fix = FixRecord {
5820 agent: agent_id,
5821 addressed: Vec::new(),
5822 rejected: Vec::new(),
5823 notes: String::new(),
5824 committed: false,
5825 failed: None,
5826 duration_ms: 0,
5827 continuation: None,
5828 };
5829 let mut final_seat = seat.clone();
5830 match out {
5831 AgentOutcome::Ok(o) => {
5832 fix.duration_ms = o.duration_ms;
5833 let parsed = verdict::extract_json::<FixReport>(&o.text);
5834 let incomplete_reason = match &parsed {
5835 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5836 "the reply parsed, but it reported a command whose own CLI \
5837 never confirmed an exit status"
5838 .to_owned(),
5839 ),
5840 Ok(_) => None,
5841 Err(e) => Some(e.to_string()),
5842 };
5843 match incomplete_reason {
5844 None => {
5845 let report = parsed.expect("checked Ok above");
5846 fix.addressed = report.addressed;
5847 fix.rejected = report.rejected;
5848 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5849 }
5850 Some(reason) => {
5851 let (resumed_seat, resolved, failure, cont) = self
5852 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5853 .await;
5854 fix.duration_ms += cont.cumulative_wait_ms;
5855 fix.continuation = Some(cont);
5856 final_seat = resumed_seat;
5857 match resolved {
5858 Some(report) => {
5859 fix.addressed = report.addressed;
5860 fix.rejected = report.rejected;
5861 fix.notes =
5862 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5863 }
5864 None => fix.failed = failure,
5865 }
5866 }
5867 }
5868 }
5869 AgentOutcome::Dropped(o) => {
5870 fix.duration_ms = o.duration_ms;
5871 let why = o
5872 .dropped
5873 .as_ref()
5874 .map(|d| d.why.as_str())
5875 .unwrap_or("the CLI ended the stream without delivering its answer");
5876 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5877 }
5878 AgentOutcome::Quota(o) => {
5879 self.state.quota.push(QuotaLoss::from_report(
5880 final_seat.key.clone(),
5881 "fix",
5882 o.quota.as_ref(),
5883 ));
5884 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5885 }
5886 AgentOutcome::Failed(e) => fix.failed = Some(e),
5887 AgentOutcome::ResourceBlocked(e) => {
5888 fix.failed = Some(format!("the fixer was not started: {e}"));
5889 }
5890 }
5891 if fix.continuation.is_none() {
5894 fix.continuation = Some(ContinuationRecord::not_needed());
5895 }
5896 self.state.seats.insert(final_seat.key.clone(), final_seat);
5897
5898 let rescue_message = format!(
5899 "magi: operator-selected fix ({}) (uncommitted work)",
5900 self.state.operator_fixes[request_index]
5901 .findings
5902 .iter()
5903 .map(|f| f.id.as_str())
5904 .collect::<Vec<_>>()
5905 .join(", ")
5906 );
5907 let repo = self.state.repo.clone();
5908 let coordination = self.coordinate(&repo).await?;
5909 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5910 self.state.note_withheld("fix", &r.withheld);
5911 }
5912 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5913 fix.committed = after != head_at_request;
5914 git::worktree_remove(&self.state.repo, &fix_worktree)
5915 .await
5916 .ok();
5917 drop(coordination);
5918
5919 self.state.event(
5920 "fix",
5921 match &fix.failed {
5922 Some(reason) => format!(
5923 "operator fix: adoption report was lost ({reason}); {}",
5924 if fix.committed {
5925 "committed"
5926 } else {
5927 "NO new commit"
5928 }
5929 ),
5930 None => format!(
5931 "operator fix: {} addressed, {} rejected, {}",
5932 fix.addressed.len(),
5933 fix.rejected.len(),
5934 if fix.committed {
5935 "committed"
5936 } else {
5937 "NO new commit"
5938 }
5939 ),
5940 },
5941 );
5942
5943 for f in &mut self.state.operator_fixes[request_index].findings {
5950 f.outcome = if fix.failed.is_some() {
5951 OperatorFixOutcome::Unreported
5952 } else if fix.addressed.contains(&f.id) {
5953 OperatorFixOutcome::Addressed
5954 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5955 OperatorFixOutcome::Rejected { why: r.why.clone() }
5956 } else {
5957 OperatorFixOutcome::Unreported
5958 };
5959 }
5960
5961 let committed = fix.committed;
5962 if committed {
5963 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5964 }
5965 self.state.operator_fixes[request_index].fix = Some(fix);
5966 self.state.save()?;
5969 if self.state.resource_block.is_some() {
5970 return self.stop_for_resource();
5971 }
5972
5973 if committed {
5974 self.state.event(
5975 "fix",
5976 format!(
5977 "operator fix committed {}; opening a follow-up review-only run",
5978 short(&after)
5979 ),
5980 );
5981 let origin =
5984 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5985 match Self::review(
5986 &self.state.repo,
5987 &winner.branch,
5988 self.state.config.clone(),
5989 origin,
5990 )
5991 .await
5992 {
5993 Ok(mut follow_up) => {
5994 follow_up.state.event(
5995 "start",
5996 format!(
5997 "requested by an operator fix on run {} for finding(s) {}",
5998 self.state.id,
5999 self.state.operator_fixes[request_index]
6000 .findings
6001 .iter()
6002 .map(|f| f.id.as_str())
6003 .collect::<Vec<_>>()
6004 .join(", "),
6005 ),
6006 );
6007 follow_up.state.save()?;
6008 let follow_up_id = follow_up.state.id.clone();
6009 let adopted = match crate::direct::adopt(
6012 &follow_up.state,
6013 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6014 ) {
6015 Ok(a) => a,
6016 Err(e) => {
6017 self.state.event(
6020 "fix",
6021 format!(
6022 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6023 ),
6024 );
6025 self.state.save()?;
6026 return Ok(());
6027 }
6028 };
6029 let executed = follow_up.execute().await;
6030 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6031 if let Some(a) = adopted {
6032 a.finish(&follow_up.state, executed);
6033 }
6034 if let Some(e) = failure {
6035 self.state.event(
6036 "fix",
6037 format!(
6038 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6039 ),
6040 );
6041 }
6042 self.state.operator_fixes[request_index].follow_up_review_run =
6043 Some(follow_up_id);
6044 }
6045 Err(e) => {
6046 self.state.event(
6047 "fix",
6048 format!("committed the fix but could not open a follow-up review: {e:#}"),
6049 );
6050 }
6051 }
6052 self.state.save()?;
6053 }
6054
6055 Ok(())
6056 }
6057
6058 async fn ask_fixer(
6078 &mut self,
6079 winner: &Candidate,
6080 node: &'static str,
6081 round: Option<usize>,
6082 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6083 ) -> (SeatJob, SeatState, AgentOutcome) {
6084 let attempts = fixer::attempts(&self.state, &self.roles, winner);
6085 self.ask_fixer_among(attempts, node, round, None, build)
6086 .await
6087 }
6088
6089 async fn ask_fixer_among(
6094 &mut self,
6095 attempts: Vec<(AgentSpec, String)>,
6096 node: &'static str,
6097 round: Option<usize>,
6098 restore: Option<(&Path, &str)>,
6099 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6100 ) -> (SeatJob, SeatState, AgentOutcome) {
6101 let run_id = self.state.id.clone();
6102 let prompts = self.state.config.prompts.clone();
6103 let cache = self.state.config.cache_dir();
6104 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6105 let mut last = None;
6106 for (i, (spec, key)) in attempts.into_iter().enumerate() {
6107 let seat = self.seat(&key, &spec.id);
6108 let mut job = build(spec.clone(), seat);
6109 if i > 0 {
6110 job.stem = format!("{}-{}", job.stem, spec.id);
6111 }
6112 let ctx = WaveCtx {
6113 carry_seats: false,
6114 pauses: Some((&self.pause, &self.interrupt)),
6115 run: &run_id,
6116 node,
6117 prompts: &prompts,
6118 cache: cache.as_deref(),
6119 round,
6120 };
6121 let (seat, out) =
6122 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6123 let advances = match &out {
6126 AgentOutcome::Ok(o) => agent::output_advances(o),
6127 AgentOutcome::ResourceBlocked(_) => false,
6128 _ => true,
6129 };
6130 if let Some(next) = ids.get(i + 1)
6131 && advances
6132 {
6133 self.state.seats.insert(seat.key.clone(), seat.clone());
6134 let class =
6135 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6136 let reason = match &out {
6137 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6138 other => fail_reason(other),
6139 };
6140 if let Some((wt, rev)) = restore
6144 && let Err(e) = self.graft_restore(wt, rev).await
6145 {
6146 let why =
6147 format!("could not restore the worktree before the next fixer: {e:#}");
6148 tracing::warn!("{why}");
6149 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6150 AgentOutcome::ResourceBlocked(why)
6151 } else {
6152 AgentOutcome::Failed(why)
6153 };
6154 last = Some((job, seat, out));
6155 break;
6156 }
6157 record_handover(
6158 &mut self.state,
6159 node,
6160 &seat.key,
6161 &spec.id,
6162 next,
6163 &class,
6164 &reason,
6165 );
6166 continue;
6167 }
6168 last = Some((job, seat, out));
6169 break;
6170 }
6171 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6172 if let AgentOutcome::Quota(output) = &out
6173 && node != "graft"
6174 {
6175 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6180 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6181 self.state.quota.push(loss);
6182 self.state.seats.insert(seat.key.clone(), seat.clone());
6183 self.state.status = RunStatus::Blocked;
6184 return (
6185 job,
6186 seat,
6187 AgentOutcome::ResourceBlocked(
6188 "fixer unavailable; restore agent availability and resume".into(),
6189 ),
6190 );
6191 }
6192 (job, seat, out)
6193 }
6194
6195 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6206 use crate::recovery::{Mode, Repair};
6207 if !self.state.reviews.is_empty() {
6208 return Ok(());
6209 }
6210 let Some(mut rec) = self.state.recovery.clone() else {
6211 return Ok(());
6212 };
6213 if rec.mode != Mode::VerifyRepair {
6214 return Ok(());
6215 }
6216 let wt = winner.worktree.clone();
6217 let mut head = git::rev_parse(&wt, "HEAD").await?;
6218 match rec.repair.clone() {
6219 Repair::Done { .. } => return Ok(()),
6220 Repair::WaitingForRestore { before } => {
6221 self.graft_restore(&wt, &before).await?;
6222 head = before;
6223 rec.repair = Repair::NotStarted;
6224 self.state.recovery = Some(rec.clone());
6225 self.state.save()?;
6226 }
6227 Repair::Started { before } => {
6228 if let Err(e) = self.graft_restore(&wt, &before).await {
6232 if e.is::<crate::repo_lock::ResourceBlocked>() {
6233 return Err(e);
6234 }
6235 return self
6236 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6237 .await;
6238 }
6239 rec.repair = Repair::Done {
6240 before: before.clone(),
6241 after: before,
6242 committed: false,
6243 failed: Some("interrupted; rolled back".to_owned()),
6244 };
6245 self.state
6246 .event("recovery", "the repair was interrupted and rolled back");
6247 self.state.recovery = Some(rec);
6248 self.state.save()?;
6249 return Ok(());
6250 }
6251 Repair::NotStarted => {}
6252 }
6253 if crate::recovery::recheck(&mut rec, &head) {
6254 self.state.event("recovery", rec.summary());
6255 self.state.recovery = Some(rec);
6256 self.state.save()?;
6257 return Ok(());
6258 }
6259 rec.repair = Repair::Started {
6260 before: head.clone(),
6261 };
6262 self.state.recovery = Some(rec.clone());
6263 self.state.save()?;
6264
6265 let language = self.state.config.graph.language.clone();
6266 let sessions = self.state.config.graph.sessions;
6267 let artifacts = agent::artifacts_dir(&self.state.dir());
6268 let fix_prompt = prompt::recovery_fix(
6269 &self.state.instruction,
6270 &rec,
6271 self.state.config.graph.review_rounds,
6272 &language,
6273 );
6274 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6275 self.state.event(
6276 "recovery",
6277 format!(
6278 "repair: the fixer starts from the failed verification on {}",
6279 short(&head)
6280 ),
6281 );
6282 let (_job, seat, out) = self
6283 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6284 prompt: fix_prompt.clone(),
6285 spec,
6286 seat,
6287 cwd: wt.clone(),
6288 timeout,
6289 allow_write: true,
6290 sessions,
6291 artifacts: artifacts.clone(),
6292 stem: "fix-recovery".to_owned(),
6293 handover: None,
6294 })
6295 .await;
6296 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6297 rec.repair = Repair::WaitingForRestore { before: head };
6298 self.state.recovery = Some(rec);
6299 return self.stop_for_resource();
6300 }
6301 let failed = match &out {
6302 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6303 AgentOutcome::Quota(o) => {
6304 self.state.quota.push(QuotaLoss::from_report(
6305 seat.key.clone(),
6306 "fix",
6307 o.quota.as_ref(),
6308 ));
6309 Some("rate limited (quota); the fixer could not run".to_owned())
6310 }
6311 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6312 AgentOutcome::Failed(e) => Some(e.clone()),
6313 };
6314 self.state.seats.insert(seat.key.clone(), seat);
6315 let coordination = self.coordinate(&wt).await?;
6316 let failed = match failed {
6317 Some(why) => {
6319 if let Err(e) = git::restore_to(&wt, &head).await {
6320 return self
6321 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6322 .await;
6323 }
6324 Some(why)
6325 }
6326 None => {
6327 if let Ok(r) = git::rescue_commit(
6328 &wt,
6329 "magi: recovery repair of the failed verification (uncommitted work)",
6330 )
6331 .await
6332 {
6333 self.state.note_withheld("fix", &r.withheld);
6334 }
6335 None
6336 }
6337 };
6338 let after = git::rev_parse(&wt, "HEAD").await?;
6339 drop(coordination);
6340 self.state.event(
6341 "recovery",
6342 match &failed {
6343 Some(why) => format!("repair: nothing kept ({why})"),
6344 None if after != head => {
6345 format!("repair: committed {}", short(&after))
6346 }
6347 None => "repair: the fixer made no commit".to_owned(),
6348 },
6349 );
6350 rec.repair = Repair::Done {
6351 before: head.clone(),
6352 committed: after != head,
6353 after,
6354 failed,
6355 };
6356 self.state.recovery = Some(rec);
6357 self.state.save()?;
6358 Ok(())
6359 }
6360
6361 async fn stop_recovery(
6364 &mut self,
6365 mut rec: crate::recovery::RecoveryRecord,
6366 head: &str,
6367 why: String,
6368 ) -> Result<()> {
6369 rec.repair = crate::recovery::Repair::Started {
6371 before: head.to_owned(),
6372 };
6373 self.state.recovery = Some(rec);
6374 self.state
6375 .event("recovery", format!("repair could not be undone: {why}"));
6376 self.state.status = RunStatus::Blocked;
6377 self.state.save()?;
6378 bail!("recovery repair could not be rolled back: {why}")
6379 }
6380
6381 async fn review_loop(&mut self) -> Result<()> {
6382 if self.hold_base_sync().await? {
6383 return Ok(());
6384 }
6385 loop {
6386 if self
6387 .state
6388 .base_sync
6389 .as_ref()
6390 .is_some_and(|s| s.conflict.is_some())
6391 {
6392 return Ok(());
6393 }
6394 if self.catchup_repairable() {
6395 let record = self.state.reviews.last().expect("repairable round").clone();
6396 let winner = self
6397 .state
6398 .winner()
6399 .cloned()
6400 .context("catch-up repair has no winner")?;
6401 if !self.fix_review_round(&winner, record).await? {
6402 return Ok(());
6403 }
6404 }
6405 self.review_loop_pass().await?;
6406 if !self.catchup_repairable() {
6407 return Ok(());
6408 }
6409 }
6410 }
6411
6412 fn catchup_repairable(&self) -> bool {
6415 self.state.reviews.len() < self.state.config.graph.review_rounds
6416 && self.state.reviews.last().is_some_and(|r| {
6417 !r.clean
6418 && r.blocking == 0
6419 && r.fix.is_none()
6420 && r.e2e_status() == E2eStatus::Failed
6421 })
6422 }
6423
6424 async fn review_loop_pass(&mut self) -> Result<()> {
6425 if self
6430 .state
6431 .base_sync
6432 .as_ref()
6433 .is_some_and(|s| s.conflict.is_some())
6434 {
6435 return Ok(());
6436 }
6437 let run_id = self.state.id.clone();
6442 let prompts = self.state.config.prompts.clone();
6443 let Some(winner) = self.state.winner().cloned() else {
6444 return Ok(());
6445 };
6446 let max_rounds = self.state.config.graph.review_rounds;
6447 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6457 if status == RunStatus::Gating {
6460 self.record_contested_handoff();
6461 }
6462 self.state.status = status;
6463 self.state.save()?;
6464 return Ok(());
6465 }
6466 self.state.status = RunStatus::Reviewing;
6467 if self.state.reviews.last().is_some_and(|r| {
6477 r.e2e_status() == E2eStatus::ResourceBlocked
6478 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6479 }) {
6480 let shell = self.state.config.shell();
6481 return self
6482 .stop_unresolved_review(
6483 "the last round's own verification never resolved",
6484 &shell,
6485 &winner.worktree,
6486 )
6487 .await;
6488 }
6489
6490 self.recovery_repair(&winner).await?;
6491 if self.resource_halt()? {
6492 return Ok(());
6493 }
6494
6495 let repo = self.state.repo.clone();
6496 let root = self.state.worktree_root();
6497 let language = self.state.config.graph.language.clone();
6498 let sessions = self.state.config.graph.sessions;
6499 let artifacts = self.merge_repair_artifacts();
6500 let base = self.landing_base();
6501 let base_short = short(&base);
6502 let reviewers = self.roles.reviewers.clone();
6503 let shell = self.state.config.shell();
6504
6505 for round in (self.state.reviews.len() + 1)..=max_rounds {
6506 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6507 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6508 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6509 let prev_verification = self
6518 .state
6519 .reviews
6520 .last()
6521 .and_then(|r| r.verification_summary(&head));
6522 let prior_section = match (&self.state.recovery, round) {
6525 (Some(rec), 1) if self.state.reviews.is_empty() => {
6526 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6527 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6528 Some(crate::recovery::head_of(
6529 &git::diff(&winner.worktree, reviewed, "HEAD")
6530 .await
6531 .unwrap_or_default(),
6532 crate::recovery::SINCE_DIFF_MAX,
6533 ))
6534 }
6535 _ => None,
6536 };
6537 Some(prompt::recovery_review(rec, since.as_deref()))
6538 }
6539 _ => None,
6540 };
6541
6542 let mut jobs = Vec::new();
6546 let coordination = self.coordinate(&repo).await?;
6547 for (r, spec) in reviewers.iter().cloned().enumerate() {
6548 let wt = root.join(format!("review-{}", r + 1));
6549 if wt.exists() {
6552 git::reset_detached(&wt, &head).await?;
6553 } else {
6554 git::worktree_add_detached(&repo, &wt, &head).await?;
6555 }
6556 self.setup_seat_worktree(&repo, &wt, false).await?;
6557 let seat_key = format!("review-{}", r + 1);
6558 let spec = pick_start_spec(
6561 &self.roles.reviewer_roster,
6562 spec,
6563 self.state.seat_history.get(&seat_key),
6564 );
6565 let seat = self.seat(&seat_key, &spec.id);
6566 jobs.push(SeatJob {
6567 prompt: prompt::review(&prompt::ReviewCtx {
6568 instruction: &self.state.instruction,
6569 branch: &winner.branch,
6570 base_short: &base_short,
6571 stat: &stat,
6572 patch: &patch,
6573 verification: prev_verification.as_ref(),
6574 prior: prior_section.as_deref(),
6575 reviewers: reviewers.len(),
6576 round,
6577 rounds: max_rounds,
6578 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6581 lens: Lens::for_seat(r),
6582 language: &language,
6583 github_language: &self.state.config.graph.github_language,
6584 }),
6585 spec,
6586 seat,
6587 cwd: wt,
6588 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6589 allow_write: false,
6590 sessions,
6591 artifacts: artifacts.clone(),
6592 stem: format!("review-{round}-{}", r + 1),
6593 handover: None,
6594 });
6595 }
6596
6597 drop(coordination);
6598 self.state.event(
6599 "review",
6600 format!(
6601 "round {round}: {} reviewers on {}",
6602 jobs.len(),
6603 short(&head)
6604 ),
6605 );
6606 let mut quota_losses = Vec::new();
6607 let review_retries = self.state.config.graph.retries;
6608 let review_cache = self.state.config.cache_dir();
6609 let ctx = WaveCtx {
6610 carry_seats: true,
6611 pauses: Some((&self.pause, &self.interrupt)),
6612 run: &run_id,
6613 node: "review",
6614 prompts: &prompts,
6615 cache: review_cache.as_deref(),
6616 round: Some(round),
6617 };
6618 let results = ask_json_wave::<Review>(
6619 jobs,
6620 Arc::clone(&self.sem),
6621 review_retries,
6622 &self.roles.reviewer_roster,
6623 &ctx,
6624 &mut quota_losses,
6625 &mut self.state,
6626 &|_: &Review| Ok(()),
6627 )
6628 .await;
6629 if self.state.resource_block.is_some() {
6630 return self.stop_for_resource();
6631 }
6632 let round_quota_missing = quota_losses.len();
6636 self.state.quota.extend(quota_losses);
6637
6638 let mut records = Vec::new();
6639 let mut all_findings = Vec::new();
6640 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6641 let agent_id = seat.agent.clone();
6642 self.state.seats.insert(seat.key.clone(), seat);
6643 let mut record = ReviewRecord {
6644 reviewer: r + 1,
6645 agent: agent_id,
6646 summary: String::new(),
6647 findings: Vec::new(),
6648 vote: None,
6649 failed: None,
6650 duration_ms: 0,
6651 attempts,
6657 };
6658 match res {
6659 Ok((review, out)) => {
6660 record.summary =
6668 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6669 record.vote = Some(review.vote);
6670 record.duration_ms = out.duration_ms;
6671 for (n, mut f) in review.findings.into_iter().enumerate() {
6672 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6675 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6676 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6677 f.file = f
6683 .file
6684 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6685 all_findings.push(f.clone());
6686 record.findings.push(f);
6687 }
6688 self.state.event(
6689 "review",
6690 format!(
6691 "round {round}: reviewer {} voted {} with {} finding(s)",
6692 r + 1,
6693 review.vote.label(),
6694 record.findings.len()
6695 ),
6696 );
6697 }
6698 Err(e) => {
6699 record.failed = Some(e.to_string());
6700 self.state.event(
6701 "review",
6702 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6703 );
6704 }
6705 }
6706 records.push(record);
6707 }
6708
6709 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6716 let vote_split =
6717 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6718 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6719 if vote_split {
6720 self.state.event(
6721 "review",
6722 format!(
6723 "round {round}: votes split ({}) — one round of reconsideration",
6724 initial_votes
6725 .iter()
6726 .map(|v| v.label())
6727 .collect::<Vec<_>>()
6728 .join(", ")
6729 ),
6730 );
6731 let panel: Vec<ReviewSeatReport<'_>> = records
6734 .iter()
6735 .filter_map(|r| {
6736 r.vote.map(|vote| ReviewSeatReport {
6737 reviewer: r.reviewer,
6738 vote,
6739 summary: &r.summary,
6740 findings: &r.findings,
6741 })
6742 })
6743 .collect();
6744
6745 let mut jobs = Vec::new();
6746 let mut seats_at = Vec::new();
6747 for (r, spec) in reviewers.iter().cloned().enumerate() {
6748 if records[r].vote.is_none() {
6752 continue;
6753 }
6754 let wt = root.join(format!("review-{}", r + 1));
6755 let seat_key = format!("review-{}", r + 1);
6756 let spec = self.occupant(&seat_key, spec);
6757 let seat = self.seat(&seat_key, &spec.id);
6758 let build = |with_patch: bool| {
6766 prompt::review_reconsider(&ReviewReconsiderCtx {
6767 instruction: &self.state.instruction,
6768 reviewer: r + 1,
6769 lens: Lens::for_seat(r),
6770 panel: &panel,
6771 patch: with_patch.then_some(ReviewPatch {
6772 branch: &winner.branch,
6773 base_short: &base_short,
6774 stat: &stat,
6775 patch: &patch,
6776 }),
6777 round,
6778 rounds: max_rounds,
6779 language: &language,
6780 })
6781 };
6782 let full = build(true);
6783 let prompt = if has_context(&spec, &seat, sessions) {
6784 build(false)
6785 } else {
6786 full.clone()
6787 };
6788 jobs.push(SeatJob {
6789 prompt,
6790 spec,
6791 seat,
6792 cwd: wt,
6793 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6794 allow_write: false,
6795 sessions,
6796 artifacts: artifacts.clone(),
6797 stem: format!("review-{round}-reconsider-{}", r + 1),
6798 handover: Some(full),
6799 });
6800 seats_at.push(r);
6801 }
6802
6803 let mut recon_quota_losses = Vec::new();
6804 let recon_cache = self.state.config.cache_dir();
6805 let recon_ctx = WaveCtx {
6806 carry_seats: true,
6807 pauses: Some((&self.pause, &self.interrupt)),
6808 run: &run_id,
6809 node: "review",
6810 prompts: &prompts,
6811 cache: recon_cache.as_deref(),
6812 round: Some(round),
6813 };
6814 let recon_results = ask_json_wave::<ReviewRevote>(
6815 jobs,
6816 Arc::clone(&self.sem),
6817 review_retries,
6818 &self.roles.reviewer_roster,
6819 &recon_ctx,
6820 &mut recon_quota_losses,
6821 &mut self.state,
6822 &|_: &ReviewRevote| Ok(()),
6823 )
6824 .await;
6825 if self.state.resource_block.is_some() {
6826 return self.stop_for_resource();
6827 }
6828 self.state.quota.extend(recon_quota_losses);
6829
6830 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6831 let agent_id = seat.agent.clone();
6832 self.state.seats.insert(seat.key.clone(), seat);
6833 let mut rec = ReviewRevoteRecord {
6834 reviewer: r + 1,
6835 agent: agent_id,
6836 vote: None,
6837 reason: String::new(),
6838 failed: None,
6839 };
6840 match res {
6841 Ok((rv, _)) => {
6842 rec.vote = Some(rv.vote);
6843 rec.reason =
6844 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6845 self.state.event(
6846 "review",
6847 format!(
6848 "round {round}: reviewer {} revoted {}",
6849 r + 1,
6850 rv.vote.label()
6851 ),
6852 );
6853 }
6854 Err(e) => {
6855 rec.failed = Some(e.to_string());
6856 self.state.event(
6857 "review",
6858 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6859 );
6860 }
6861 }
6862 reconsideration.push(rec);
6863 }
6864 } else if initial_votes.len() > 1 {
6865 self.state.event(
6866 "review",
6867 format!(
6868 "round {round}: votes agreed ({}) — no reconsideration",
6869 initial_votes[0].label()
6870 ),
6871 );
6872 }
6873
6874 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6875 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6876 let defer_e2e =
6887 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6888 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6889 let reason =
6890 format!("{blocking} blocking finding(s) already required a fix this round");
6891 self.state.event(
6892 "verify",
6893 format!(
6894 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6895 {}); it will run once a round has none left",
6896 short(&head)
6897 ),
6898 );
6899 (Vec::new(), false, true, Some(reason))
6900 } else {
6901 let e2e_commands = self.state.config.verify.e2e.clone();
6902 let cache_dir = self.state.config.cache_dir();
6903 let context = format!("round {round}");
6904 let wait = self.repository_wait();
6905 let (e2e, verify_retried) = with_cache_lease(
6906 &mut self.state,
6907 cache_dir.as_deref(),
6908 "e2e",
6909 "e2e",
6910 &winner.worktree,
6911 &head,
6912 verify_timeout,
6913 &context,
6914 &wait,
6915 |state, budget| {
6916 let shell = shell.clone();
6917 let e2e_commands = e2e_commands.clone();
6918 let worktree = winner.worktree.clone();
6919 let context = context.clone();
6920 async move {
6921 run_e2e_with_retry(
6922 state,
6923 &shell,
6924 &e2e_commands,
6925 &worktree,
6926 budget,
6927 &context,
6928 )
6929 .await
6930 }
6931 },
6932 )
6933 .await?;
6934 (e2e, verify_retried, false, None)
6935 };
6936
6937 let expected = records.len();
6938 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6939 let incomplete = answered < expected;
6940 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6941 let policy = self.state.config.graph.incomplete_review;
6942 let clean = round_is_clean(
6943 blocking,
6944 e2e_ok,
6945 answered,
6946 expected,
6947 round_quota_missing,
6948 policy,
6949 );
6950
6951 let mut round_record = ReviewRound {
6952 round,
6953 head: head.clone(),
6954 verified_head: None,
6955 verified_at: None,
6956 reviews: records,
6957 e2e,
6958 verify_retried,
6959 e2e_deferred,
6960 e2e_defer_reason,
6961 fix: None,
6962 blocking,
6963 answered,
6964 expected,
6965 clean,
6966 progressed: false,
6967 vote_split,
6968 reconsideration,
6969 verdict: None,
6970 };
6971 round_record.verdict = ReviewVote::worst(
6975 round_record
6976 .final_votes()
6977 .into_iter()
6978 .map(|(_, _, vote)| vote),
6979 );
6980 if !matches!(
6991 round_record.e2e_status(),
6992 E2eStatus::Deferred | E2eStatus::NotConfigured
6993 ) {
6994 round_record.verified_head = Some(head.clone());
6995 round_record.verified_at = Some(Timestamp::now());
6996 }
6997
6998 if incomplete {
6999 let missing: Vec<String> = round_record
7000 .reviews
7001 .iter()
7002 .filter(|r| r.failed.is_some())
7003 .map(|r| format!("review-{}", r.reviewer))
7004 .collect();
7005 self.state.event(
7006 "review",
7007 format!(
7008 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7009 missing.join(", ")
7010 ),
7011 );
7012 }
7013
7014 if clean {
7015 self.state.event(
7016 "review",
7017 if incomplete && policy == IncompleteReviewPolicy::Warn {
7018 format!(
7019 "round {round}: clean (warn policy, incomplete panel) — no \
7020 blocking findings from the seats that answered, verification green"
7021 )
7022 } else if incomplete {
7023 format!(
7024 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7025 quorum) — no blocking findings from the seats that answered, \
7026 verification green",
7027 expected - answered
7028 )
7029 } else {
7030 format!("round {round}: clean — no blocking findings, verification green")
7031 },
7032 );
7033 self.state.reviews.push(round_record);
7034 self.state.status = RunStatus::Gating;
7035 self.state.save()?;
7036 return Ok(());
7037 }
7038
7039 if incomplete && blocking == 0 && e2e_ok {
7047 self.state.reviews.push(round_record);
7048 self.state.save()?;
7049 if round == max_rounds {
7050 self.state.status = RunStatus::Blocked;
7051 self.state.event(
7052 "review",
7053 format!(
7054 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7055 refusing to call it clean",
7056 expected - answered
7057 ),
7058 );
7059 return Ok(());
7060 }
7061 continue;
7062 }
7063
7064 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7076 self.state.reviews.push(round_record);
7077 return self
7078 .stop_unresolved_review(
7079 "the round's own verification could not run",
7080 &shell,
7081 &winner.worktree,
7082 )
7083 .await;
7084 }
7085
7086 if round == max_rounds {
7087 self.state.reviews.push(round_record);
7088 return self
7089 .stop_reviewing(
7090 &format!(
7091 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7092 ),
7093 &shell,
7094 &winner.worktree,
7095 )
7096 .await;
7097 }
7098
7099 if !self.fix_review_round(&winner, round_record).await? {
7100 return Ok(());
7101 }
7102 }
7103 Ok(())
7104 }
7105
7106 async fn fix_review_round(
7109 &mut self,
7110 winner: &Candidate,
7111 mut round_record: ReviewRound,
7112 ) -> Result<bool> {
7113 let round = round_record.round;
7114 let shell = self.state.config.shell();
7115 let step = self.run_fix_round(winner, &mut round_record).await?;
7116 if matches!(step, FixStep::Resource) {
7117 self.stop_for_resource()?;
7118 return Ok(false);
7119 }
7120 if self.state.reviews.len() < round {
7121 self.state.reviews.push(round_record);
7122 } else {
7123 self.state.reviews[round - 1] = round_record;
7124 }
7125 self.state.save()?;
7126
7127 if matches!(step, FixStep::Recorded { report_lost: true }) {
7140 self.stop_reviewing(
7141 "the fixer's adoption report never came back, even after resuming its \
7142 own seat; refusing to start another round against the same worktree \
7143 while that is unresolved",
7144 &shell,
7145 &winner.worktree,
7146 )
7147 .await?;
7148 return Ok(false);
7149 }
7150
7151 let streak = self
7152 .state
7153 .reviews
7154 .iter()
7155 .rev()
7156 .take_while(|r| !r.progressed)
7157 .count();
7158 if streak >= STAGNANT_LIMIT {
7159 self.stop_reviewing(
7160 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7161 &shell,
7162 &winner.worktree,
7163 )
7164 .await?;
7165 return Ok(false);
7166 }
7167 Ok(true)
7168 }
7169
7170 async fn run_fix_round(
7172 &mut self,
7173 winner: &Candidate,
7174 round_record: &mut ReviewRound,
7175 ) -> Result<FixStep> {
7176 let round = round_record.round;
7177 let max_rounds = self.state.config.graph.review_rounds;
7178 let run_id = self.state.id.clone();
7179 let prompts = self.state.config.prompts.clone();
7180 let language = self.state.config.graph.language.clone();
7181 let sessions = self.state.config.graph.sessions;
7182 let artifacts = self.merge_repair_artifacts();
7183 let base = self.landing_base();
7184 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7185 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7186 let this_round_verification = round_record.verification_summary(&head);
7187 let all_findings: Vec<_> = round_record
7188 .reviews
7189 .iter()
7190 .flat_map(|r| r.findings.iter().cloned())
7191 .collect();
7192 let blocking_findings: Vec<_> = all_findings
7195 .iter()
7196 .filter(|f| f.severity.blocks())
7197 .cloned()
7198 .collect();
7199 let fix_prompt = prompt::fix_in(
7200 &self.state.instruction,
7201 &blocking_findings,
7202 this_round_verification.as_ref(),
7203 round,
7204 max_rounds,
7205 &language,
7206 &self.state.config.graph.github_language,
7207 );
7208 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7209 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7210 let (job, seat, out) = self
7211 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7212 prompt: fix_prompt.clone(),
7213 spec,
7214 seat,
7215 cwd: winner.worktree.clone(),
7216 timeout,
7217 allow_write: true,
7218 sessions,
7219 artifacts: artifacts.clone(),
7220 stem: format!("fix-{round}"),
7221 handover: None,
7222 })
7223 .await;
7224 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7227 return Ok(FixStep::Resource);
7228 }
7229 let agent_id = seat.agent.clone();
7230
7231 let mut fix = FixRecord {
7232 agent: agent_id,
7233 addressed: Vec::new(),
7234 rejected: Vec::new(),
7235 notes: String::new(),
7236 committed: false,
7237 failed: None,
7238 duration_ms: 0,
7239 continuation: None,
7240 };
7241 let mut continuation = ContinuationRecord::not_needed();
7242 let mut final_seat = seat.clone();
7243 match out {
7244 AgentOutcome::Ok(o) => {
7245 fix.duration_ms = o.duration_ms;
7246 let parsed = verdict::extract_json::<FixReport>(&o.text);
7247 let incomplete_reason = match &parsed {
7254 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7255 "the reply parsed, but it reported a command whose own CLI never \
7256 confirmed an exit status"
7257 .to_owned(),
7258 ),
7259 Ok(_) => None,
7260 Err(e) => Some(e.to_string()),
7261 };
7262 match incomplete_reason {
7263 None => {
7264 let report = parsed.expect("checked Ok above");
7265 fix.addressed = report.addressed;
7266 fix.rejected = report.rejected;
7267 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7268 }
7269 Some(reason) => {
7270 let (resumed_seat, resolved, failure, cont) = self
7271 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7272 .await;
7273 fix.duration_ms += cont.cumulative_wait_ms;
7274 continuation = cont;
7275 final_seat = resumed_seat;
7276 match resolved {
7277 Some(report) => {
7278 fix.addressed = report.addressed;
7279 fix.rejected = report.rejected;
7280 fix.notes =
7281 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7282 }
7283 None => fix.failed = failure,
7284 }
7285 }
7286 }
7287 }
7288 AgentOutcome::Dropped(o) => {
7290 fix.duration_ms = o.duration_ms;
7291 let why = o
7292 .dropped
7293 .as_ref()
7294 .map(|d| d.why.as_str())
7295 .unwrap_or("the CLI ended the stream without delivering its answer");
7296 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7297 }
7298 AgentOutcome::Quota(o) => {
7299 self.state.quota.push(QuotaLoss::from_report(
7300 final_seat.key.clone(),
7301 "fix",
7302 o.quota.as_ref(),
7303 ));
7304 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7305 }
7306 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7307 }
7308 if self.state.resource_block.is_some() {
7309 self.state.seats.insert(final_seat.key.clone(), final_seat);
7310 return Ok(FixStep::Resource);
7311 }
7312 fix.continuation = Some(continuation);
7313 self.state.seats.insert(final_seat.key.clone(), final_seat);
7314 let coordination = self.coordinate(&winner.worktree).await?;
7315 if let Ok(r) = git::rescue_commit(
7316 &winner.worktree,
7317 &format!("magi: review round {round} fixes (uncommitted work)"),
7318 )
7319 .await
7320 {
7321 self.state.note_withheld("fix", &r.withheld);
7322 }
7323 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7324 fix.committed = after != before;
7325 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7333 drop(coordination);
7334 let progressed = diff_after != patch;
7335 let commit_note = if fix.committed {
7336 "committed"
7337 } else {
7338 "NO new commit"
7339 };
7340 let tree_note = if progressed {
7341 "changed vs base"
7342 } else {
7343 "unchanged vs base"
7344 };
7345 self.state.event(
7346 "fix",
7347 match &fix.failed {
7348 Some(reason) => {
7354 format!(
7355 "round {round}: fixer's adoption report was lost ({reason}); \
7356 {commit_note}, tree {tree_note}"
7357 )
7358 }
7359 None => format!(
7360 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7361 {tree_note}{}",
7362 fix.addressed.len(),
7363 fix.rejected.len(),
7364 if continuation.outcome == ContinuationOutcome::Resumed {
7365 format!(
7366 " (adoption report recovered after {} continuation(s))",
7367 continuation.attempts
7368 )
7369 } else {
7370 String::new()
7371 },
7372 ),
7373 },
7374 );
7375 round_record.fix = Some(fix);
7376 round_record.progressed = progressed;
7377 Ok(FixStep::Recorded {
7378 report_lost: matches!(
7379 continuation.outcome,
7380 ContinuationOutcome::Exhausted
7381 | ContinuationOutcome::QuotaLost
7382 | ContinuationOutcome::NoSession
7383 ),
7384 })
7385 }
7386
7387 fn record_contested_handoff(&mut self) {
7421 if self.state.contested_handoff.is_some() {
7422 return;
7423 }
7424 let Some(contested) = self
7425 .state
7426 .reviews
7427 .last()
7428 .and_then(ReviewRound::contested_handoff)
7429 else {
7430 return;
7431 };
7432 self.state.event(
7433 "review",
7434 format!(
7435 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7436 wait for the owner's approval",
7437 contested.findings.len(),
7438 contested.rejecters.len()
7439 ),
7440 );
7441 self.state.contested_handoff = Some(contested);
7442 }
7443
7444 async fn stop_unresolved_review(
7445 &mut self,
7446 why: &str,
7447 shell: &[String],
7448 worktree: &Path,
7449 ) -> Result<()> {
7450 self.stop_reviewing_inner(why, shell, worktree, true).await
7451 }
7452
7453 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7454 self.stop_reviewing_inner(why, shell, worktree, false).await
7455 }
7456
7457 async fn stop_reviewing_inner(
7458 &mut self,
7459 why: &str,
7460 shell: &[String],
7461 worktree: &Path,
7462 repairable: bool,
7463 ) -> Result<()> {
7464 let round_idx = self.state.reviews.len() - 1;
7465 let needs_catchup_run = matches!(
7473 self.state.reviews[round_idx].e2e_status(),
7474 E2eStatus::Deferred | E2eStatus::ResourceBlocked
7475 );
7476 if needs_catchup_run {
7477 let round = self.state.reviews[round_idx].round;
7478 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7479 let commands = self.state.config.verify.e2e.clone();
7480 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7481 let cache_dir = self.state.config.cache_dir();
7482 let context = format!(
7483 "round {round}: verification unresolved, catching up before the final decision"
7484 );
7485 let wait = self.repository_wait();
7486 let (outcomes, verify_retried) = with_cache_lease(
7487 &mut self.state,
7488 cache_dir.as_deref(),
7489 "e2e",
7490 "e2e",
7491 worktree,
7492 &attempted_head,
7493 timeout,
7494 &context,
7495 &wait,
7496 |state, budget| {
7497 let shell = shell.to_vec();
7498 let commands = commands.clone();
7499 let context = context.clone();
7500 async move {
7501 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7502 .await
7503 }
7504 },
7505 )
7506 .await?;
7507 let last = &mut self.state.reviews[round_idx];
7508 last.e2e = outcomes;
7509 last.verify_retried = verify_retried;
7510 last.verified_head = Some(attempted_head);
7517 last.verified_at = Some(Timestamp::now());
7518 if verify_inconclusive(&last.e2e) {
7519 self.state.save()?;
7526 return Ok(());
7527 }
7528 last.e2e_deferred = false;
7529 }
7530 let last = &self.state.reviews[round_idx];
7531 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7532
7533 match last.e2e_status() {
7534 E2eStatus::Failed => {
7535 let red: Vec<String> = last
7536 .e2e
7537 .iter()
7538 .filter(|o| !o.ok())
7539 .map(|o| {
7540 format!(
7541 "`{}` -> {:?}\n{}",
7542 o.command,
7543 o.code,
7544 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7545 )
7546 })
7547 .collect();
7548 let can_repair = last.blocking == 0 && last.fix.is_none();
7549 self.state.reviews[round_idx].clean = false;
7550 self.state
7551 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7552 self.state.status = RunStatus::Blocked;
7553 if repairable
7554 && can_repair
7555 && self.state.reviews.len() < self.state.config.graph.review_rounds
7556 {
7557 self.state.status = RunStatus::Reviewing;
7558 }
7559 }
7560 E2eStatus::ResourceBlocked => {
7565 self.state.event(
7566 "review",
7567 format!(
7568 "{why}; e2e could not run (resource unavailable); not \
7569 deciding yet"
7570 ),
7571 );
7572 }
7573 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7574 self.state.event(
7575 "review",
7576 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7577 );
7578 self.record_contested_handoff();
7579 self.state.status = RunStatus::Gating;
7580 }
7581 }
7582 self.state.save()?;
7583 Ok(())
7584 }
7585
7586 async fn gate(&mut self) -> Result<()> {
7589 if self.hold_base_sync().await? {
7590 return Ok(());
7591 }
7592 if self.state.status == RunStatus::Failed
7604 || self
7605 .state
7606 .base_sync
7607 .as_ref()
7608 .is_some_and(|s| s.conflict.is_some())
7609 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7610 != Some(RunStatus::Gating)
7611 {
7612 return Ok(());
7613 }
7614 let Some(winner) = self.state.winner().cloned() else {
7615 return Ok(());
7616 };
7617 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7618 self.state.reconcile_verification_head(&head);
7619 self.state.save()?;
7620 let wait = self.repository_wait();
7621 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7622 return Ok(());
7623 }
7624 if self.state.gate_ran {
7625 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7636 self.state.status = RunStatus::Blocked;
7637 self.state.save()?;
7638 }
7639 return Ok(());
7640 }
7641 self.state.status = RunStatus::Gating;
7642 let mut outcomes = self.run_gate(&winner).await?;
7643 let mut attempted_head = self.state.gate_head.clone();
7644 loop {
7645 if verify_inconclusive(&outcomes) {
7656 self.state.save()?;
7657 return Ok(());
7658 }
7659 if outcomes.iter().all(CommandOutcome::ok) {
7660 break;
7661 }
7662 let fixes_before = self.state.gate_fixes.len();
7663 match self.gate_fix_round(&winner, &outcomes).await? {
7664 GateFix::Retry => {
7665 outcomes = self.run_gate(&winner).await?;
7666 attempted_head = self.state.gate_head.clone();
7667 }
7668 GateFix::Stop => {
7669 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7670 if self.state.gate_fixes.len() > fixes_before
7671 && attempted_head.as_deref() != Some(current.as_str())
7672 && verify_current_head(&mut self.state, &winner, &wait).await?
7673 {
7674 outcomes = self.run_gate(&winner).await?;
7677 attempted_head = self.state.gate_head.clone();
7678 } else {
7679 break;
7680 }
7681 }
7682 GateFix::Defer => {
7683 self.state.save()?;
7684 return Ok(());
7685 }
7686 }
7687 }
7688 let passed = outcomes.iter().all(CommandOutcome::ok);
7689 self.state.gate = outcomes;
7690 self.state.gate_ran = true;
7691 self.state.gate_head = attempted_head;
7692 self.state.save()?;
7693 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7694 return Ok(());
7695 }
7696 if !passed {
7697 self.state.status = RunStatus::Blocked;
7698 let spent = self.state.gate_fixes.len();
7699 self.state.event(
7700 "gate",
7701 if spent == 0 {
7702 "gate failed; not merging".to_owned()
7703 } else {
7704 format!("gate failed after {spent} gate-fix round(s); not merging")
7705 },
7706 );
7707 }
7708 self.state.save()?;
7709 Ok(())
7710 }
7711
7712 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7722 let commands = state.config.verify.pre_gate.clone();
7723 if commands.is_empty() {
7724 return;
7725 }
7726 let shell = state.config.shell();
7727 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7728 let (outcomes, _) = run_commands(
7729 state,
7730 "pre_gate",
7731 "pre_gate",
7732 0,
7733 &shell,
7734 &commands,
7735 &winner.worktree,
7736 timeout,
7737 )
7738 .await;
7739 for o in &outcomes {
7740 if !o.ok() {
7741 tracing::warn!(
7742 "pre_gate `{}` failed ({:?}); the gate decides",
7743 o.command,
7744 o.code
7745 );
7746 }
7747 state.event(
7748 "pre_gate",
7749 format!(
7750 "`{}` -> {}",
7751 o.command,
7752 if o.ok() {
7753 "pass".to_owned()
7754 } else {
7755 format!(
7756 "FAIL ({:?})\n{}",
7757 o.code,
7758 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7759 )
7760 }
7761 ),
7762 );
7763 }
7764 state.pre_gate = outcomes;
7765 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7766 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7767 Ok(head) => {
7768 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7769 state.pre_gate_commit = Some(head);
7770 }
7771 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7772 },
7773 Ok(false) => {}
7774 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7775 }
7776 if let Err(e) = state.save() {
7777 tracing::warn!("could not persist the pre_gate record: {e:#}");
7778 }
7779 }
7780
7781 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7784 let wait = self.repository_wait();
7785 run_landing_gate(&mut self.state, winner, &wait).await
7786 }
7787
7788 async fn gate_fix_round(
7801 &mut self,
7802 winner: &Candidate,
7803 outcomes: &[CommandOutcome],
7804 ) -> Result<GateFix> {
7805 let cap = self.state.config.graph.gate_fix_rounds;
7806 let spent = self.state.gate_fixes.len();
7807 if spent >= cap {
7808 if cap > 0 {
7809 self.state.event(
7810 "gate",
7811 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7812 );
7813 }
7814 return Ok(GateFix::Stop);
7815 }
7816 if !gate_fixable(outcomes) {
7817 self.state.event(
7818 "gate",
7819 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7820 command or similar); not spending a fix round on it",
7821 );
7822 return Ok(GateFix::Stop);
7823 }
7824 let min_free = self.state.config.disk.min_free_bytes;
7825 if min_free > 0 {
7826 match crate::disk::free_bytes(&winner.worktree) {
7827 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7828 Ok(free) => {
7829 self.state.event(
7830 "gate",
7831 format!(
7832 "only {free} bytes free ({min_free} required by `[disk] \
7833 min_free_bytes`); not spending a fix round on a failure the disk \
7834 may explain"
7835 ),
7836 );
7837 return Ok(GateFix::Stop);
7838 }
7839 Err(e) => {
7840 self.state.event(
7841 "gate",
7842 format!("free disk space could not be measured ({e:#}); no fix round"),
7843 );
7844 return Ok(GateFix::Stop);
7845 }
7846 }
7847 }
7848
7849 let attempt = spent + 1;
7850 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7851 let base = self.landing_base();
7852 let fix_prompt = prompt::gate_fix_in(
7853 &self.state.instruction,
7854 &failed,
7855 attempt,
7856 cap,
7857 &self.state.config.graph.language,
7858 &self.state.config.graph.github_language,
7859 );
7860 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7861 let sessions = self.state.config.graph.sessions;
7862 let artifacts = self.merge_repair_artifacts();
7863 self.state.event(
7864 "gate",
7865 format!("gate failed; gate-fix round {attempt} of {cap}"),
7866 );
7867 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7868 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7869 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7872 head: before.clone(),
7873 passed: false,
7874 outcomes: Vec::new(),
7875 });
7876 self.state.save()?;
7877 let (_, seat, out) = self
7878 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7879 prompt: fix_prompt.clone(),
7880 spec,
7881 seat,
7882 cwd: winner.worktree.clone(),
7883 timeout,
7884 allow_write: true,
7885 sessions,
7886 artifacts: artifacts.clone(),
7887 stem: format!("gate-fix-{attempt}"),
7888 handover: None,
7889 })
7890 .await;
7891 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7894 return Ok(GateFix::Defer);
7895 }
7896 let mut record = GateFixRecord {
7897 agent: seat.agent.clone(),
7898 failed,
7899 notes: String::new(),
7900 committed: false,
7901 error: None,
7902 };
7903 match out {
7904 AgentOutcome::Ok(o) => {
7905 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7908 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7909 }
7910 }
7911 AgentOutcome::Dropped(_) => {
7912 record.error = Some("the CLI dropped the stream".to_owned());
7913 }
7914 AgentOutcome::Quota(o) => {
7915 self.state.quota.push(QuotaLoss::from_report(
7916 seat.key.clone(),
7917 "gate-fix",
7918 o.quota.as_ref(),
7919 ));
7920 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7921 }
7922 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7923 }
7924 self.state.seats.insert(seat.key.clone(), seat);
7925 let coordination = self.coordinate(&winner.worktree).await?;
7926 if let Ok(r) = git::rescue_commit(
7927 &winner.worktree,
7928 &format!("magi: gate fix {attempt} (uncommitted work)"),
7929 )
7930 .await
7931 {
7932 self.state.note_withheld("gate-fix", &r.withheld);
7933 }
7934 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7935 record.committed = after != before;
7936 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7937 let note = record.error.clone();
7938 self.state.gate_fixes.push(record);
7939 self.state.save()?;
7940 if !changed {
7941 self.state.event(
7942 "gate",
7943 match note {
7944 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7945 None => format!("gate-fix round {attempt}: the tree did not change"),
7946 },
7947 );
7948 return Ok(GateFix::Stop);
7949 }
7950 self.state.event(
7951 "gate",
7952 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7953 );
7954
7955 drop(coordination);
7956 let wait = self.repository_wait();
7957 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7958 }
7959
7960 async fn guarded_pr_message(
7967 &mut self,
7968 winner: &Candidate,
7969 facts: Option<&BranchFacts>,
7970 posting: bool,
7971 ) -> Result<Option<PrMessage>> {
7972 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7973 let head = match final_head.as_deref() {
7974 Some(h) => crate::pr_outcome::FinalHead::At(h),
7975 None => crate::pr_outcome::FinalHead::Unknown,
7976 };
7977 let summarized = if posting {
7979 Some(self.summarize_change(winner, final_head.as_deref()).await)
7980 } else {
7981 None
7982 };
7983 if self.state.resource_block.is_some() {
7988 self.state.event(
7989 "pr-summary",
7990 "a shared resource was busy; nothing was published, resume to summarize again",
7991 );
7992 return Ok(None);
7993 }
7994 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7995 anyhow::bail!(
7996 "the final commit moved during public description generation; retry before publishing"
7997 );
7998 }
7999 let note = match (&summarized, &self.state.change_summary) {
8000 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8001 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8002 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8003 (None, _) => ChangeNote::NotAsked,
8004 };
8005 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8006 if !posting {
8007 let identity = crate::scrub::Identity::current();
8008 return Ok(Some(PrMessage {
8009 title: crate::scrub::scrub(&pr.title, &identity),
8010 body: crate::scrub::scrub(&pr.body, &identity),
8011 }));
8012 }
8013 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8016 if let Some(known) = self
8017 .state
8018 .github_text
8019 .clone()
8020 .filter(|g| g.fingerprint == raw_fp)
8021 {
8022 return self.settle_withheld(winner, &pr, known).await;
8023 }
8024 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8025 let mut violations =
8026 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8027 let identity = crate::scrub::Identity::current();
8028 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8029 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8030 && !violations.contains(&crate::github_text::Violation::SensitiveData)
8031 {
8032 violations.push(crate::github_text::Violation::SensitiveData);
8033 }
8034 let has_language = violations
8038 .iter()
8039 .any(|v| *v != crate::github_text::Violation::SensitiveData);
8040 if self.state.config.graph.github_text_guard && !violations.is_empty() {
8041 self.state.event(
8042 "github-text",
8043 if has_language {
8044 format!("description rejected: {violations:?}; requesting one rewrite")
8045 } else {
8046 format!("description rejected: {violations:?}; asking the owner")
8047 },
8048 );
8049 let mut rewritten = false;
8050 let mut rejected_rewrite = None;
8051 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8052 {
8053 let output = text;
8054 if let Some(title) = summary_title(&output) {
8055 let raw_body = summary_without_title(&output);
8058 let clean_title = crate::scrub::scrub(&title, &identity);
8059 let mut body = crate::scrub::scrub(&raw_body, &identity);
8060 let redacted = clean_title != title || body != raw_body;
8061 let title = clean_title;
8062 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8063 if !complete {
8064 self.state.event(
8065 "github-text",
8066 "description rewrite omitted required findings",
8067 );
8068 }
8069 if !body.trim().is_empty() {
8070 body.push_str("\n\n");
8071 body.push_str(&public_pr_evidence(&self.state, head));
8072 }
8073 let marker = format!("magi:run/{}", self.state.id);
8074 if !body.trim().is_empty() && !body.contains(&marker) {
8075 body.push_str(&format!("\n\n{marker}"));
8076 }
8077 let rewrite = PrMessage {
8078 title: title.clone(),
8079 body: body.clone(),
8080 };
8081 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8082 let mut failed = crate::github_text::check_for(
8083 &self.state.config.graph,
8084 &title,
8085 &body,
8086 redecision,
8087 );
8088 if (!crate::github_text::shareable(&title)
8089 || !crate::github_text::shareable(&body))
8090 && !failed.contains(&crate::github_text::Violation::SensitiveData)
8091 {
8092 failed.push(crate::github_text::Violation::SensitiveData);
8093 }
8094 if complete
8095 && !title.trim().is_empty()
8096 && !body.trim().is_empty()
8097 && !failed.is_empty()
8098 {
8099 rejected_rewrite = Some(crate::run::GithubTextRewrite {
8100 title: title.clone(),
8101 body: body.clone(),
8102 failed: failed
8103 .iter()
8104 .map(|v| crate::github_text::category(*v).to_owned())
8105 .collect(),
8106 });
8107 }
8108 if complete
8109 && !title.trim().is_empty()
8110 && !raw_body.trim().is_empty()
8111 && failed.is_empty()
8112 {
8113 decision = redecision;
8114 pr = PrMessage { title, body };
8115 rewritten = true;
8116 self.state
8117 .event("github-text", "description rewrite passed");
8118 if redacted {
8119 self.state.event(
8120 "github-text",
8121 "sensitive data removed from description rewrite",
8122 );
8123 }
8124 }
8125 }
8126 }
8127 if !rewritten {
8128 self.state.event(
8129 "github-text",
8130 "description rewrite unavailable or rejected; asking the owner",
8131 );
8132 if let Some(w) =
8133 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8134 {
8135 self.retire_github_text_question("the withheld text changed");
8138 let record = crate::run::GithubTextAsk {
8139 rewrite: rejected_rewrite,
8140 fingerprint: raw_fp,
8141 title: w.title,
8142 body: w.body,
8143 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8144 question: None,
8145 asks: 0,
8146 resolved: false,
8147 chosen_title: None,
8148 choice: None,
8149 };
8150 return self.settle_withheld(winner, &pr, record).await;
8151 }
8152 }
8153 }
8154 let (title, body) =
8155 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8156 Ok(Some(PrMessage { title, body }))
8157 }
8158
8159 async fn summarize_change(
8170 &mut self,
8171 winner: &Candidate,
8172 head: Option<&str>,
8173 ) -> std::result::Result<(), String> {
8174 let Some(head) = head else {
8175 return Err("the final commit could not be read".to_owned());
8176 };
8177 let remote = self.state.config.merge.remote.clone();
8178 let tracking = format!("{remote}/{}", self.state.base_branch);
8179 if !git::rev_exists(&winner.worktree, &tracking).await {
8180 return Err(format!("the remote base `{tracking}` could not be read"));
8181 }
8182 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8183 Ok(b) if !b.is_empty() => b,
8184 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8185 };
8186 let evidence = crate::pr_outcome::summary_evidence(
8187 &self.state,
8188 crate::pr_outcome::FinalHead::At(head),
8189 );
8190 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8191 if self
8192 .state
8193 .change_summary
8194 .as_ref()
8195 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8196 {
8197 return Ok(());
8198 }
8199 if self.state.change_summary.take().is_some() {
8200 self.state.event(
8201 "pr-summary",
8202 "the final diff or evidence changed; the earlier change summary was discarded",
8203 );
8204 }
8205 if let Err(e) = write_artifact(
8207 &self.state,
8208 &format!("public-description-{fingerprint}.md"),
8209 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8210 ) {
8211 tracing::warn!("could not save public description evidence: {e:#}");
8212 }
8213 let chain = match agent::pick_chain(
8214 &self.state.config.agents,
8215 self.state.config.roles.synthesizer.as_ref(),
8216 &agent::installed,
8217 "synthesizer",
8218 ) {
8219 Ok(chain) => chain,
8220 Err(e) => {
8221 tracing::warn!("no agent for the change summary: {e:#}");
8222 self.state.event("pr-summary", "no summary agent available");
8223 return Err("no agent was available to summarize it".to_owned());
8224 }
8225 };
8226 let subjects = git::subjects(&winner.worktree, &base, head)
8227 .await
8228 .unwrap_or_default();
8229 let stat = git::diff_stat(&winner.worktree, &base, head)
8230 .await
8231 .unwrap_or_default();
8232 let stat: String = stat.chars().take(6_000).collect();
8233 let mut prompt = crate::prompt::change_summary_in(
8234 &base,
8235 head,
8236 &subjects,
8237 stat.trim(),
8238 &self.state.config.graph.github_language,
8239 );
8240 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8241 let sessions = self.state.config.graph.sessions;
8242 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8243 let prompts = self.state.config.prompts.clone();
8244 let cache = self.state.config.cache_dir();
8245 let run = self.state.id.clone();
8246 for (i, spec) in chain.into_iter().enumerate() {
8247 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8248 let github_spec = match agent::for_github_text(
8249 &spec,
8250 &self.state.config.graph.github_language,
8251 &winner.worktree,
8252 ) {
8253 Ok(spec) => spec,
8254 Err(e) => {
8255 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8256 continue;
8257 }
8258 };
8259 let job = SeatJob {
8260 spec: github_spec,
8261 seat,
8262 cwd: winner.worktree.clone(),
8263 prompt: prompt.clone(),
8264 timeout,
8265 allow_write: false,
8266 sessions,
8267 artifacts: agent::artifacts_dir(&self.state.dir()),
8268 stem: if i == 0 {
8269 "change-summary".to_owned()
8270 } else {
8271 format!("change-summary-{}", spec.id)
8272 },
8273 handover: None,
8274 };
8275 let ctx = WaveCtx {
8276 run: &run,
8277 node: "pr-summary",
8278 prompts: &prompts,
8279 cache: cache.as_deref(),
8280 round: None,
8281 carry_seats: false,
8282 pauses: Some((&self.pause, &self.interrupt)),
8283 };
8284 let (seat, outcome) =
8285 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8286 self.state.seats.insert(seat.key.clone(), seat);
8287 let text = match outcome {
8288 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8289 AgentOutcome::ResourceBlocked(why) => {
8290 if self.state.resource_block.is_none() {
8293 self.state.resource_block = Some(crate::run::ResourceBlock {
8294 cause: crate::run::ResourceBlockCause::Other,
8295 node: "pr-summary".to_owned(),
8296 kind: "shared resource".to_owned(),
8297 detail: why,
8298 at: Timestamp::now(),
8299 });
8300 }
8301 return Err("a shared resource was busy".to_owned());
8302 }
8303 _ => continue,
8304 };
8305 let blind = &self.state.config.blind;
8306 let Some(title) = summary_title(&text) else {
8307 continue;
8308 };
8309 let body: String = summary_without_title(&text)
8310 .lines()
8311 .filter(|l| !l.contains("magi:run/"))
8312 .collect::<Vec<_>>()
8313 .join("\n");
8314 let body = blind::sanitize_prose(body.trim(), blind);
8315 let title = blind::sanitize_prose(&title, blind);
8316 if body.trim().is_empty() || title.trim().is_empty() {
8317 continue;
8318 }
8319 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8320 continue;
8321 }
8322 if git::rev_parse(&winner.worktree, "HEAD")
8324 .await
8325 .ok()
8326 .as_deref()
8327 != Some(head)
8328 {
8329 return Err("the branch moved while the summary was written".to_owned());
8330 }
8331 if git::merge_base(&winner.worktree, &tracking, head)
8332 .await
8333 .ok()
8334 .as_deref()
8335 != Some(base.as_str())
8336 || crate::pr_outcome::summary_evidence(
8337 &self.state,
8338 crate::pr_outcome::FinalHead::At(head),
8339 ) != evidence
8340 {
8341 return Err("the summary evidence changed while it was written".to_owned());
8342 }
8343 self.state.change_summary = Some(crate::run::ChangeSummary {
8344 head: head.to_owned(),
8345 base,
8346 evidence: fingerprint,
8347 title,
8348 body,
8349 });
8350 self.state.event(
8351 "pr-summary",
8352 format!("change summary written by {} for the final diff", spec.id),
8353 );
8354 if let Err(e) = self.state.save() {
8355 tracing::warn!("could not save the change summary: {e:#}");
8356 }
8357 return Ok(());
8358 }
8359 self.state
8360 .event("pr-summary", "no agent produced a change summary");
8361 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8362 }
8363
8364 async fn rewrite_pr_text(
8374 &mut self,
8375 winner: &Candidate,
8376 pr: &PrMessage,
8377 violations: &[crate::github_text::Violation],
8378 ) -> Option<String> {
8379 let prompt = format!(
8380 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8381 pr.title, pr.body
8382 );
8383 let sessions = self.state.config.graph.sessions;
8384 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8385 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8388 let key = format!("impl-{}", winner.label);
8389 if let Some(spec) = self
8390 .state
8391 .config
8392 .agents
8393 .iter()
8394 .find(|a| a.id == winner.agent)
8395 .cloned()
8396 && let Some(existing) = self.state.seats.get(&key)
8397 && existing.agent == spec.id
8398 && agent::has_session(spec.kind, existing, sessions)
8399 {
8400 attempts.push((spec, Some(existing.clone())));
8401 }
8402 match agent::pick_chain(
8403 &self.state.config.agents,
8404 self.state.config.roles.language_judge.as_ref(),
8405 &agent::installed,
8406 "language_judge",
8407 ) {
8408 Ok(chain) => {
8409 for spec in chain {
8410 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8411 continue;
8412 }
8413 attempts.push((spec, None));
8414 }
8415 }
8416 Err(e) if attempts.is_empty() => {
8417 tracing::warn!("no agent for the description rewrite: {e:#}");
8418 self.state
8419 .event("github-text", "no rewrite agent available");
8420 return None;
8421 }
8422 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8423 }
8424 let prompts = self.state.config.prompts.clone();
8425 let cache = self.state.config.cache_dir();
8426 let run = self.state.id.clone();
8427 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8428 let resumed = resumed_seat.is_some();
8429 let seat = resumed_seat.unwrap_or_else(|| {
8430 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8431 });
8432 let github_spec = match agent::for_github_text(
8433 &spec,
8434 &self.state.config.graph.github_language,
8435 &winner.worktree,
8436 ) {
8437 Ok(spec) => spec,
8438 Err(e) => {
8439 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8440 continue;
8441 }
8442 };
8443 let job = SeatJob {
8444 spec: github_spec,
8445 seat,
8446 cwd: winner.worktree.clone(),
8447 prompt: prompt.clone(),
8448 timeout: retry_budget(timeout_full, resumed),
8449 allow_write: false,
8450 sessions,
8451 artifacts: agent::artifacts_dir(&self.state.dir()),
8452 stem: if i == 0 {
8453 "github-text-rewrite".to_owned()
8454 } else {
8455 format!("github-text-rewrite-{}", spec.id)
8456 },
8457 handover: None,
8458 };
8459 let ctx = WaveCtx {
8460 run: &run,
8461 node: "github-text",
8462 prompts: &prompts,
8463 cache: cache.as_deref(),
8464 round: None,
8465 carry_seats: false,
8466 pauses: Some((&self.pause, &self.interrupt)),
8467 };
8468 let (seat, outcome) =
8469 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8470 self.state.seats.insert(seat.key.clone(), seat);
8471 match outcome {
8472 AgentOutcome::Ok(o)
8473 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8474 {
8475 return Some(o.text);
8476 }
8477 AgentOutcome::ResourceBlocked(_) => return None,
8478 _ => {}
8479 }
8480 }
8481 None
8482 }
8483
8484 fn retire_github_text_question(&mut self, why: &str) {
8486 let Some(id) = self
8487 .state
8488 .github_text
8489 .as_ref()
8490 .and_then(|g| g.question.clone())
8491 else {
8492 return;
8493 };
8494 let _ = ask::Questions::open().update(&id, |q| {
8495 q.abandon(why);
8496 Ok(())
8497 });
8498 }
8499
8500 async fn settle_withheld(
8504 &mut self,
8505 winner: &Candidate,
8506 pr: &PrMessage,
8507 mut g: crate::run::GithubTextAsk,
8508 ) -> Result<Option<PrMessage>> {
8509 use crate::github_text as gt;
8510 let store = ask::Questions::open();
8511 let timeout = self.state.config.graph.answer_timeout;
8512 if !g.resolved {
8513 let mut standing = g
8514 .question
8515 .as_ref()
8516 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8517 if let Some(q) = standing.as_ref()
8518 && q.status.open()
8519 && gt::expired(q, timeout)
8520 {
8521 let why = format!("no answer within {}s of asking", timeout.max(1));
8522 standing = store
8523 .update(&q.id, |q| {
8524 q.abandon(&why);
8525 Ok(())
8526 })
8527 .ok()
8528 .map(|(q, ())| q);
8529 }
8530 let mut ask_again: Option<Option<Vec<String>>> = None;
8531 match standing {
8532 None if g.asks <= 1 => {
8535 g.asks = 0;
8536 ask_again = Some(None);
8537 }
8538 None => {
8539 g.resolved = true;
8540 g.chosen_title = None;
8541 g.choice = Some(GithubTextChoice::Fallback);
8542 }
8543 Some(q) if q.status.open() => {
8544 self.park_on_github_text(&g)?;
8545 return Ok(None);
8546 }
8547 Some(q) => match gt::read_reply(&q) {
8548 gt::Reply::Fallback => {
8549 g.resolved = true;
8550 g.chosen_title = None;
8551 g.choice = Some(GithubTextChoice::Fallback);
8552 }
8553 gt::Reply::Original => {
8554 g.resolved = true;
8555 g.chosen_title = None;
8556 g.choice = Some(GithubTextChoice::Original);
8557 }
8558 gt::Reply::Rewrite => {
8559 g.resolved = true;
8560 g.chosen_title = None;
8561 g.choice = Some(if g.rewrite.is_some() {
8562 GithubTextChoice::Rewrite
8563 } else {
8564 GithubTextChoice::Fallback
8565 });
8566 }
8567 gt::Reply::Title(text) => {
8568 let first = text
8569 .lines()
8570 .map(str::trim)
8571 .find(|l| !l.is_empty())
8572 .unwrap_or("");
8573 let decision = if first.is_empty() {
8574 None
8575 } else {
8576 let candidate = PrMessage {
8577 title: first.to_owned(),
8578 body: String::new(),
8579 };
8580 self.judge_pr_language(&winner.worktree, &candidate).await
8581 };
8582 match gt::vet_title(
8583 &text,
8584 decision,
8585 self.state.config.graph.github_english(),
8586 ) {
8587 Ok(title) => {
8588 g.resolved = true;
8589 g.chosen_title = Some(title);
8590 g.choice = Some(GithubTextChoice::MyTitle);
8591 }
8592 Err(cats) if g.asks < 2 => {
8593 ask_again = Some(Some(
8594 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8595 ))
8596 }
8597 Err(_) => {
8598 g.resolved = true;
8599 g.chosen_title = None;
8600 g.choice = Some(GithubTextChoice::Fallback);
8601 }
8602 }
8603 }
8604 },
8605 }
8606 if let Some(rejected) = ask_again {
8607 let w = gt::Withheld {
8610 title: g.title,
8611 body: g.body,
8612 categories: g.categories.clone(),
8613 };
8614 let artifact = match crate::run::write_artifact(
8617 &self.state,
8618 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8619 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8620 ) {
8621 Ok(p) => Some(p),
8622 Err(e) => {
8623 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8624 None
8625 }
8626 };
8627 let (detail, comparison) = gt::question_content(
8628 &w,
8629 &pr.title,
8630 &pr.body,
8631 &self.state.id,
8632 rejected.as_deref(),
8633 artifact.as_deref(),
8634 g.rewrite.as_ref(),
8635 );
8636 let mut q = ask::Question::new(
8637 self.state.id.clone(),
8638 gt::ASK_NODE.to_owned(),
8639 gt::ASK_SEAT.to_owned(),
8640 gt::question_summary(&self.state.config.graph.language, &w),
8641 detail,
8642 gt::question_choices(&w, g.rewrite.as_ref()),
8643 );
8644 q.posting_comparison = Some(comparison);
8645 q.answer_timeout = timeout;
8646 g.question = Some(q.id.clone());
8649 g.asks += 1;
8650 self.state.github_text = Some(g.clone());
8651 self.state.event(
8652 "github-text",
8653 format!("asking the owner about the withheld text ({})", q.short()),
8654 );
8655 self.state.save()?;
8656 store
8657 .put_executable(&mut q)
8658 .context("file the posting-gate question")?;
8659 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8660 tracing::warn!(
8661 "could not notify about posting-gate question {}: {e:#}",
8662 q.short()
8663 );
8664 }
8665 self.park_on_github_text(&g)?;
8666 return Ok(None);
8667 }
8668 }
8669 let resolved = g.chosen_title.clone();
8670 let original = g.choice == Some(GithubTextChoice::Original);
8671 self.state.event(
8673 "github-text",
8674 if original {
8675 format!(
8676 "posting the original text for: {}",
8677 match (g.title, g.body) {
8678 (true, true) => "title, description",
8679 (true, false) => "title",
8680 _ => "description",
8681 }
8682 )
8683 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8684 "posting the saved rewrite title and description".to_owned()
8685 } else if resolved.is_some() {
8686 "posting the owner's replacement title".to_owned()
8687 } else {
8688 "posting the neutral text for the withheld fields".to_owned()
8689 },
8690 );
8691 self.state.github_text = Some(g.clone());
8692 self.state.save()?;
8693 if g.choice == Some(GithubTextChoice::Rewrite)
8694 && let Some(r) = g.rewrite.as_ref()
8695 {
8696 return Ok(Some(PrMessage {
8697 title: r.title.clone(),
8698 body: r.body.clone(),
8699 }));
8700 }
8701 let w = gt::Withheld {
8702 title: g.title,
8703 body: g.body,
8704 categories: g.categories.clone(),
8705 };
8706 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8707 if let Some(t) = resolved {
8708 title = t;
8709 }
8710 let english = gt::LanguageDecision {
8712 title_english: true,
8713 body_english: true,
8714 body_complete: true,
8715 };
8716 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8717 if original {
8718 if g.title {
8721 title = pr.title.clone();
8722 }
8723 if g.body {
8724 body = pr.body.clone();
8725 }
8726 }
8727 Ok(Some(PrMessage { title, body }))
8728 }
8729
8730 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8731 self.state.github_text = Some(g.clone());
8732 self.state.end_pass_parked(ParkCause::Question);
8733 self.state.parked = true;
8734 self.state.event(
8735 "github-text",
8736 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8737 );
8738 self.state.save()
8739 }
8740
8741 async fn judge_pr_language(
8744 &mut self,
8745 cwd: &Path,
8746 pr: &PrMessage,
8747 ) -> Option<crate::github_text::LanguageDecision> {
8748 self.state.config.roles.language_judge.as_ref()?;
8749 if !self.state.config.graph.language_gate() {
8750 return None;
8751 }
8752 let decision =
8753 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8754 self.state.event(
8755 "github-text",
8756 if decision.is_some() {
8757 "language decided by the language judge"
8758 } else {
8759 "language judge unavailable; using built-in heuristics"
8760 },
8761 );
8762 decision
8763 }
8764
8765 async fn merge(&mut self) -> Result<()> {
8766 if self.resource_halt()? || self.hold_base_sync().await? {
8767 return Ok(());
8768 }
8769 if self
8784 .state
8785 .base_sync
8786 .as_ref()
8787 .is_some_and(|s| s.conflict.is_some())
8788 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8789 != Some(RunStatus::Gating)
8790 || !self.state.gate_allows_progress()
8798 {
8799 return Ok(());
8800 }
8801 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8810 return Ok(());
8811 }
8812 let Some(winner) = self.state.winner().cloned() else {
8813 return Ok(());
8814 };
8815 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8816 if self.state.gate_head.as_deref() != Some(head.as_str()) {
8817 self.state.status = RunStatus::Blocked;
8818 self.state.event(
8819 "gate",
8820 "winner changed after gate; fresh verification required",
8821 );
8822 self.state.save()?;
8823 return Ok(());
8824 }
8825 let wait = self.repository_wait();
8826 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8827 return Ok(());
8828 }
8829 let repo = self.state.repo.clone();
8830 let base = self.state.base_branch.clone();
8831 let mode = self.state.config.merge.mode;
8832 let style = self.state.config.merge.style;
8833 let facts = if is_review_run(&self.state) {
8834 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8835 let start = review_base(
8836 &repo,
8837 &self.state.config.merge.remote,
8838 &base,
8839 &self.state.base_commit,
8840 &winner.branch,
8841 )
8842 .await;
8843 branch_facts(&repo, &start, &winner.branch).await
8844 } else {
8845 None
8846 };
8847 let leaked = if is_review_run(&self.state) {
8850 leaked_subjects(&self.state, &winner.branch).await
8851 } else {
8852 Some(Vec::new())
8853 };
8854 let Some(pr) = self
8858 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8859 .await?
8860 else {
8861 return Ok(());
8862 };
8863 let message = pr.commit_message();
8864
8865 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
8866 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
8867 return Ok(());
8868 }
8869 let outcome = match mode {
8870 MergeMode::None => MergeOutcome {
8871 mode,
8872 ok: true,
8873 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8874 empty: false,
8875 },
8876 MergeMode::Pr | MergeMode::Local
8877 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8878 {
8879 MergeOutcome {
8880 mode,
8881 ok: false,
8882 detail: empty_candidate_detail(&self.state, &base),
8883 empty: true,
8884 }
8885 }
8886 MergeMode::Local => {
8887 let on = git::current_branch(&repo).await?;
8888 if on.as_deref() != Some(base.as_str()) {
8889 MergeOutcome {
8890 mode,
8891 ok: false,
8892 detail: format!(
8893 "{} has {} checked out, not the base branch {base}",
8894 repo.display(),
8895 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8896 ),
8897 empty: false,
8898 }
8899 } else if !git::is_clean(&repo).await? {
8900 MergeOutcome {
8901 mode,
8902 ok: false,
8903 detail: format!("{} is dirty; refusing to merge", repo.display()),
8904 empty: false,
8905 }
8906 } else {
8907 let out = match style {
8908 MergeStyle::Merge => {
8909 git::merge_no_ff(&repo, &winner.branch, &message).await?
8910 }
8911 MergeStyle::Squash => {
8912 git::merge_squash(&repo, &winner.branch, &message).await?
8913 }
8914 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8915 };
8916 MergeOutcome {
8917 mode,
8918 ok: out.ok(),
8919 detail: if out.ok() { out.stdout } else { out.stderr },
8920 empty: false,
8921 }
8922 }
8923 }
8924 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8925 self.state.pr_publication_pending = true;
8926 self.state.save()?;
8927 let out =
8928 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8929 self.state.pr_publication_pending = out.is_err();
8930 MergeOutcome {
8931 mode,
8932 ok: out.is_ok(),
8933 detail: match out {
8934 Ok(url) => url,
8935 Err(e) => format!("{e:#}"),
8936 },
8937 empty: false,
8938 }
8939 }
8940 MergeMode::Pr => {
8941 let remote = self.state.config.merge.remote.clone();
8942 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8943 let pushed = if self.state.foreign_checkout.is_some() {
8944 let refspec = format!("{delivery_head}:refs/heads/{public}");
8945 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
8946 } else {
8947 git::push(&winner.worktree, &remote, &winner.branch).await?
8948 };
8949 if !pushed.ok() {
8950 MergeOutcome {
8951 mode,
8952 ok: false,
8953 detail: pushed.stderr,
8954 empty: false,
8955 }
8956 } else {
8957 if self.state.foreign_checkout.is_some() {
8958 self.state.pr_pushed_head = Some(delivery_head.clone());
8959 }
8960 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
8966 let out = match pr_merge_plan(found) {
8967 PrPlan::Create => {
8968 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
8969 .await
8970 }
8971 PrPlan::Adopt { url, title } => {
8972 self.state
8973 .event("merge", format!("Pr: adopted open pull request {url}"));
8974 if self.state.github_text.as_ref().is_some_and(|g| {
8975 g.resolved
8976 && g.choice == Some(GithubTextChoice::Rewrite)
8977 && g.rewrite.is_some()
8978 }) {
8979 land::set_pr_text_verbatim(
8982 &winner.worktree,
8983 &url,
8984 &pr.title,
8985 &pr.body,
8986 )
8987 .await
8988 .map(|()| url)
8989 } else {
8990 if title != pr.title
8991 && (!is_review_run(&self.state)
8992 || leaked
8993 .as_deref()
8994 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8995 && let Err(e) =
8996 if self.state.github_text.as_ref().is_some_and(|g| {
8997 g.resolved
8998 && g.title
8999 && g.choice == Some(GithubTextChoice::Original)
9000 }) {
9001 land::set_pr_title_verbatim(
9003 &winner.worktree,
9004 &url,
9005 &pr.title,
9006 )
9007 .await
9008 } else {
9009 land::set_pr_title(
9010 &mut self.state,
9011 &winner.worktree,
9012 &url,
9013 &pr.title,
9014 )
9015 .await
9016 }
9017 {
9018 tracing::warn!("could not refresh title of {url}: {e:#}");
9019 self.state
9020 .event("merge", format!("Pr: title refresh failed: {e:#}"));
9021 }
9022 Ok(url)
9023 }
9024 }
9025 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9026 };
9027 match out {
9028 Ok(url) => MergeOutcome {
9029 mode,
9030 ok: true,
9031 detail: url,
9032 empty: false,
9033 },
9034 Err(e) => MergeOutcome {
9035 mode,
9036 ok: false,
9037 detail: e.to_string(),
9038 empty: false,
9039 },
9040 }
9041 }
9042 }
9043 };
9044
9045 self.state.status = match (mode, outcome.ok) {
9046 (MergeMode::None, _) => RunStatus::Ready,
9047 (_, true) if outcome.empty => RunStatus::Ready,
9048 (MergeMode::Pr, true) => RunStatus::Ready,
9049 (_, true) => RunStatus::Merged,
9050 (_, false) => RunStatus::Blocked,
9051 };
9052 self.state.event(
9053 "merge",
9054 format!(
9055 "{:?}: {}",
9056 mode,
9057 outcome.detail.lines().next().unwrap_or("")
9058 ),
9059 );
9060 self.state.merge = Some(outcome);
9061 self.state.save()?;
9062
9063 if self.state.config.graph.land
9069 && mode == MergeMode::Pr
9070 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9071 {
9072 self.run_land().await?;
9073 }
9074 self.settle_questions();
9079 Ok(())
9080 }
9081
9082 async fn run_land(&mut self) -> Result<()> {
9093 crate::pr_head::ensure_owner(&self.state)?;
9094 let url = self
9095 .state
9096 .pr
9097 .as_ref()
9098 .map(|pr| pr.url.clone())
9099 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9100 .unwrap_or_default();
9101 let url = url.lines().next().unwrap_or("").trim().to_owned();
9102 if !url.starts_with("http") {
9103 return Ok(());
9104 }
9105 loop {
9108 match land::land(&mut self.state, &url).await {
9109 Ok(_)
9110 if self
9111 .state
9112 .merge_repairs
9113 .last()
9114 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9115 {
9116 if self.repair_merge_review().await? {
9117 continue;
9118 }
9119 }
9120 Ok(_) if self.state.resource_block.is_some() => {
9121 return self.stop_for_resource();
9122 }
9123 Ok(pr) if self.state.parked => {
9124 let _ = pr;
9128 }
9129 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9130 self.state.status = RunStatus::Superseded;
9131 self.state.save()?;
9132 }
9133 Ok(pr) => {
9134 self.state.status = match pr.state {
9135 land::PrLifecycle::Merged => RunStatus::Merged,
9136 _ => RunStatus::Blocked,
9137 };
9138 if bump::should_release_bump(self.state.status)
9145 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9146 {
9147 self.state
9153 .event("bump", format!("release bump skipped: {e:#}"));
9154 }
9155 if self.state.status == RunStatus::Merged {
9158 crate::followup::after_merge(&mut self.state, &pr.url).await;
9159 }
9160 self.state.save()?;
9161 }
9162 Err(e) => {
9163 if e.is::<crate::repo_lock::ResourceBlocked>() {
9164 return Err(e);
9165 }
9166 self.state.status = RunStatus::Blocked;
9167 self.state.event("land", format!("gave up: {e}"));
9168 self.state.save()?;
9169 }
9170 }
9171 break;
9172 }
9173 Ok(())
9174 }
9175
9176 async fn repair_merge_review(&mut self) -> Result<bool> {
9179 use crate::run::MergeRepairPhase as Phase;
9180 crate::pr_head::ensure_owner(&self.state)?;
9181 let i = self.state.merge_repairs.len() - 1;
9182 let request = self.state.merge_repairs[i].clone();
9183 if request.phase == Phase::Fixing {
9184 let restored = match self.state.winner().cloned() {
9185 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9186 None => Err(anyhow::anyhow!("repair has no winning branch")),
9187 };
9188 let why = match restored {
9189 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9190 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9191 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9192 };
9193 self.hold_merge_repair(i, why)?;
9194 return Ok(false);
9195 }
9196 if request.phase == Phase::Reviewing && self.state.gate_ran {
9201 let winner = self
9202 .state
9203 .winner()
9204 .context("repair has no winning branch")?;
9205 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9206 if self.state.gate_head.as_deref() != Some(head.as_str())
9207 || !git::is_clean(&winner.worktree).await?
9208 {
9209 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9210 return Ok(false);
9211 }
9212 }
9213 let failure = if i >= self.state.config.graph.land_rounds
9214 || self.state.config.graph.review_rounds == 0
9215 {
9216 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9217 } else {
9218 None
9219 };
9220 if let Some(why) = failure {
9221 self.hold_merge_repair(i, why)?;
9222 return Ok(false);
9223 }
9224 if request.phase == Phase::Requested {
9225 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9226 Ok(pr) => pr,
9227 Err(e) => {
9228 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9229 return Ok(false);
9230 }
9231 };
9232 self.state.merge_repairs[i].phase = Phase::Fixing;
9233 self.state.land_approval = None;
9234 self.state.land_armed_head = None;
9235 self.state.save()?;
9236 let evidence = serde_json::to_string_pretty(&request)?;
9239 let reason = format!(
9240 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9241 );
9242 let budget = self.state.config.graph.land_rounds;
9243 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9244 match result {
9245 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9246 Ok(land::Fixed::Deferred) => {
9247 let worktree = self
9251 .state
9252 .winner()
9253 .context("repair has no winning branch")?
9254 .worktree
9255 .clone();
9256 self.graft_restore(&worktree, &request.head).await?;
9257 self.state.merge_repairs[i].phase = Phase::Requested;
9258 self.stop_for_resource()?;
9259 return Ok(false);
9260 }
9261 Ok(land::Fixed::Failed(why)) => {
9262 self.hold_merge_repair(
9263 i,
9264 format!(
9265 "repair failed: {why}; inspect the branch and run an explicit review"
9266 ),
9267 )?;
9268 return Ok(false);
9269 }
9270 Err(e) => {
9271 self.hold_merge_repair(
9272 i,
9273 format!(
9274 "repair failed: {e:#}; inspect the branch and run an explicit review"
9275 ),
9276 )?;
9277 return Ok(false);
9278 }
9279 }
9280 self.begin_merge_repair_review(i)?;
9284 }
9285 self.review_loop().await?;
9286 self.gate().await?;
9287 if self.resource_halt()? || self.state.parked {
9288 return Ok(false);
9289 }
9290 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9291 != Some(RunStatus::Gating)
9292 || !self.state.gate_allows_progress()
9293 {
9294 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9295 return Ok(false);
9296 }
9297 let winner = self
9300 .state
9301 .winner()
9302 .cloned()
9303 .context("repair has no winning branch")?;
9304 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9305 if self.state.gate_head.as_deref() != Some(after.as_str())
9306 || !git::is_clean(&winner.worktree).await?
9307 {
9308 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9309 return Ok(false);
9310 }
9311 let expected = self
9312 .state
9313 .pr_pushed_head
9314 .as_deref()
9315 .unwrap_or(&request.head)
9316 .to_owned();
9317 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9318 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9321 return Ok(false);
9322 }
9323 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9324 let refspec = format!("{after}:refs/heads/{public}");
9325 let push = git::git_raw(
9326 &winner.worktree,
9327 &["push", &lease, &self.state.config.merge.remote, &refspec],
9328 )
9329 .await;
9330 match push {
9331 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9332 Ok(out) => {
9333 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9334 return Ok(false);
9335 }
9336 Err(e) => {
9337 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9338 return Ok(false);
9339 }
9340 }
9341 self.state.merge_repairs[i].phase = Phase::Complete;
9342 self.state.status = RunStatus::Landing;
9343 self.state.event(
9344 "repair",
9345 "new review and gate completed; fresh owner merge decision required",
9346 );
9347 self.state.save()?;
9348 Ok(true)
9349 }
9350
9351 fn merge_repair_artifacts(&self) -> PathBuf {
9352 let artifacts = agent::artifacts_dir(&self.state.dir());
9353 match self
9354 .state
9355 .merge_repairs
9356 .last()
9357 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9358 {
9359 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9360 None => artifacts,
9361 }
9362 }
9363
9364 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9365 if let Some(e2e) = self.state.head_e2e.take() {
9366 self.state
9367 .verification_history
9368 .push(crate::run::VerificationEvidence {
9369 superseded_by: None,
9370 archived_at: Some(Timestamp::now()),
9371 fixes_spent: Some(self.state.gate_fixes.len()),
9372 gate_head: None,
9373 gate_ran: false,
9374 gate: Vec::new(),
9375 e2e: Some(e2e),
9376 });
9377 }
9378 self.state.reviews.clear();
9379 self.state.gate.clear();
9380 self.state.gate_ran = false;
9381 self.state.gate_head = None;
9382 self.state.gate_fixes.clear();
9383 self.state.gate_repair_e2e = None;
9384 self.state.pre_gate.clear();
9385 self.state.pre_gate_commit = None;
9386 self.state.contested_handoff = None;
9387 self.state.land_reruns = None;
9388 self.state.land_approval = None;
9389 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9390 self.state.status = RunStatus::Reviewing;
9391 self.state.save()
9392 }
9393
9394 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9395 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9396 self.state.merge_repairs[index].failure = Some(why.clone());
9397 self.state.status = RunStatus::Blocked;
9398 self.state.event("repair", why);
9399 self.state.save()
9400 }
9401
9402 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9406 if let Some(existing) = self.state.seats.get(key)
9407 && existing.agent == agent
9408 {
9409 return existing.clone();
9410 }
9411 let fresh = if self.state.seats.contains_key(key) {
9414 handover_seat(key, agent, self.state.next_seat_seed())
9415 } else {
9416 SeatState::new(key, agent, self.state.seed)
9417 };
9418 self.state.seats.insert(key.to_owned(), fresh.clone());
9419 fresh
9420 }
9421
9422 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9428 match self.state.seats.get(key) {
9429 Some(s) if s.agent != spec.id => {
9430 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9431 }
9432 _ => spec,
9433 }
9434 }
9435
9436 fn view(&self, c: &Candidate) -> CandidateView {
9438 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9439 .unwrap_or_default();
9440 let (patch, _) = blind::sanitize_patch(
9441 &format!("candidate {} patch", c.label),
9442 &raw,
9443 &self.state.config.blind,
9444 );
9445 CandidateView {
9446 label: c.label,
9447 branch: c.branch.clone(),
9448 summary: c.summary.clone(),
9449 stat: c.stat.clone(),
9450 patch,
9451 }
9452 }
9453
9454 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9456 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9457 prompt::judge(
9458 "(see above)",
9459 &views,
9460 self.roles.judges.len(),
9461 base_short,
9462 "en",
9463 )
9464 }
9465
9466 fn vote_prompt_full(
9472 &self,
9473 j: usize,
9474 labels: &[char],
9475 language: &str,
9476 candidates: &[Candidate],
9477 base_short: &str,
9478 ) -> String {
9479 let mut text = format!(
9480 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9481 prompt::final_vote(labels, language),
9482 self.state.instruction,
9483 self.candidate_block(candidates, base_short)
9484 );
9485 if let Some(own) = self
9486 .state
9487 .judgements
9488 .get(j)
9489 .filter(|r| !r.ranking.is_empty())
9490 {
9491 let reasons = own
9492 .reasons
9493 .iter()
9494 .map(|(k, v)| format!("- {k}: {v}"))
9495 .collect::<Vec<_>>()
9496 .join("\n");
9497 text.push_str(&format!(
9498 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9499 own.ranking.iter().collect::<String>(),
9500 if reasons.is_empty() {
9501 ""
9502 } else {
9503 ", because:\n"
9504 }
9505 ));
9506 }
9507 if !self.state.deliberation.is_empty() {
9508 text.push_str("\n# What was argued before this vote\n");
9509 for t in self.transcript(&[], j) {
9510 text.push_str(&format!(
9511 "\n## {}{}\n\n{}\n",
9512 t.who,
9513 if t.is_self { " (you)" } else { "" },
9514 t.body.trim()
9515 ));
9516 }
9517 }
9518 text
9519 }
9520
9521 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9528 let mut turns = Vec::new();
9529 for j in &self.state.judgements {
9530 if j.ranking.is_empty() {
9531 continue;
9532 }
9533 let reasons = j
9534 .reasons
9535 .iter()
9536 .map(|(k, v)| format!("- {k}: {v}"))
9537 .collect::<Vec<_>>()
9538 .join("\n");
9539 turns.push(Turn {
9540 who: format!("Judge {} (opening ranking)", j.judge),
9541 is_self: j.judge == self_idx + 1,
9542 body: format!(
9543 "Ranked {}{}{reasons}",
9544 j.ranking.iter().collect::<String>(),
9545 if reasons.is_empty() {
9546 ""
9547 } else {
9548 ", because:\n"
9549 }
9550 ),
9551 });
9552 }
9553 for t in self
9554 .state
9555 .deliberation
9556 .iter()
9557 .flat_map(|r| r.turns.iter())
9558 .chain(current)
9559 {
9560 turns.push(Turn {
9561 who: format!("Judge {}", t.judge),
9562 is_self: t.judge == self_idx + 1,
9563 body: t.body.clone(),
9564 });
9565 }
9566 turns
9567 }
9568}
9569
9570fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9572 agent::has_session(spec.kind, seat, sessions)
9573}
9574
9575fn next_untried_in_roster<'a>(
9596 roster: &'a [AgentSpec],
9597 start: usize,
9598 tried: &BTreeSet<String>,
9599) -> Option<&'a AgentSpec> {
9600 roster
9601 .get(start + 1..)?
9602 .iter()
9603 .find(|s| !tried.contains(&s.id))
9604}
9605
9606fn pick_successor<'a>(
9627 roster: &'a [AgentSpec],
9628 start: usize,
9629 tried: &BTreeSet<String>,
9630 carried: Option<&BTreeSet<String>>,
9631 others: &BTreeSet<String>,
9632) -> Option<&'a AgentSpec> {
9633 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9634 roster
9635 .get(start + 1..)
9636 .and_then(|tail| {
9637 tail.iter()
9638 .filter(free)
9639 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9640 })
9641 .or_else(|| {
9642 carried
9643 .is_some()
9644 .then(|| roster.iter().find(free))
9645 .flatten()
9646 })
9647 .or_else(|| match carried {
9648 Some(c) => next_for_seat(roster, start, tried, c),
9649 None => next_untried_in_roster(roster, start, tried),
9650 })
9651}
9652
9653fn next_for_seat<'a>(
9663 roster: &'a [AgentSpec],
9664 start: usize,
9665 round_tried: &BTreeSet<String>,
9666 carried_failed: &BTreeSet<String>,
9667) -> Option<&'a AgentSpec> {
9668 roster
9669 .get(start + 1..)?
9670 .iter()
9671 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9672 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9673}
9674
9675fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9681 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9682 return spec;
9683 };
9684 let ok = |id: &str| !h.failed.contains(id);
9685 if let Some(last) = h.last_ok.as_deref()
9686 && ok(last)
9687 && let Some(s) = roster.iter().find(|s| s.id == last)
9688 {
9689 return s.clone();
9690 }
9691 if ok(&spec.id) {
9692 return spec;
9693 }
9694 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9695 roster
9696 .get(from + 1..)
9697 .into_iter()
9698 .flatten()
9699 .chain(roster.iter())
9700 .find(|s| ok(&s.id))
9701 .cloned()
9702 .unwrap_or(spec)
9703}
9704
9705pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9709 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9710}
9711
9712const TIMED_OUT: &str = "timed out";
9715
9716impl FailClass {
9717 fn of(out: &AgentOutcome) -> Option<Self> {
9719 match out {
9720 AgentOutcome::Ok(_) => None,
9721 AgentOutcome::Quota(_) => Some(Self::Quota),
9722 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9723 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9724 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9725 AgentOutcome::ResourceBlocked(_) => None,
9728 }
9729 }
9730
9731 fn stem_word(&self) -> &'static str {
9733 match self {
9734 Self::Quota => "quota",
9735 _ => "handover",
9736 }
9737 }
9738}
9739
9740fn failure_signature(msg: &str) -> String {
9744 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9745 if let Some(code) = line
9748 .strip_prefix("process exited with no usable output (exit code ")
9749 .and_then(|s| s.strip_suffix(')'))
9750 .and_then(|s| s.parse::<i32>().ok())
9751 {
9752 line = format!("exited with some({code}) and no usable output");
9753 } else if matches!(
9754 line.as_str(),
9755 "process exited with no usable output (no exit code reported)"
9756 | "process exited with no usable output (exit code unavailable)"
9757 ) {
9758 line = "exited with none and no usable output".to_owned();
9759 }
9760 let mut out = Vec::new();
9761 for word in line.split_whitespace() {
9762 if word.contains('/') || word.contains('\\') {
9763 out.push("<path>".to_owned());
9764 continue;
9765 }
9766 let mut w = String::new();
9767 let mut in_digits = false;
9768 for c in word.chars() {
9769 if c.is_ascii_digit() {
9770 if !in_digits {
9771 w.push('#');
9772 }
9773 in_digits = true;
9774 } else {
9775 in_digits = false;
9776 w.push(c);
9777 }
9778 }
9779 out.push(w);
9780 }
9781 out.join(" ").chars().take(120).collect()
9782}
9783
9784fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9791 match cur {
9792 FailClass::Quota | FailClass::Timeout => true,
9793 FailClass::Other(_) => prev != Some(cur),
9794 }
9795}
9796
9797fn fail_reason(out: &AgentOutcome) -> String {
9799 match out {
9800 AgentOutcome::Ok(_) => String::new(),
9801 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9802 AgentOutcome::Dropped(o) => format!(
9803 "the CLI dropped the stream ({})",
9804 o.dropped
9805 .as_ref()
9806 .map(|d| d.why.as_str())
9807 .unwrap_or("it ended without delivering its answer")
9808 ),
9809 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9810 e.lines().next().unwrap_or("").chars().take(160).collect()
9811 }
9812 }
9813}
9814
9815pub(crate) fn record_handover(
9818 state: &mut RunState,
9819 node: &str,
9820 seat: &str,
9821 from: &str,
9822 to: &str,
9823 class: &FailClass,
9824 reason: &str,
9825) {
9826 let message = if *class == FailClass::Quota {
9827 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9828 } else {
9829 format!("{seat}: handed over {from} -> {to} ({reason})")
9830 };
9831 state.event(node, message);
9832 state.handovers.push(Handover {
9833 at: Timestamp::now(),
9834 node: node.to_owned(),
9835 seat: seat.to_owned(),
9836 from: from.to_owned(),
9837 to: to.to_owned(),
9838 reason: reason.to_owned(),
9839 });
9840}
9841
9842fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9856 commands.iter().any(|c| c.exit_code.is_none())
9857}
9858
9859fn verified_noop_claim(
9872 usable: bool,
9873 commands: &[agent::CommandEvidence],
9874 text: &str,
9875) -> Option<String> {
9876 (usable && !has_unconfirmed_command(commands))
9877 .then(|| verdict::verified_noop(text))
9878 .flatten()
9879}
9880
9881fn short(commit: &str) -> String {
9882 commit.chars().take(7).collect()
9883}
9884
9885fn make_executable(path: &Path) -> Result<()> {
9886 #[cfg(unix)]
9887 {
9888 use std::os::unix::fs::PermissionsExt as _;
9889 let mut perms = std::fs::metadata(path)?.permissions();
9890 perms.set_mode(0o755);
9891 std::fs::set_permissions(path, perms)?;
9892 }
9893 #[cfg(not(unix))]
9894 {
9895 let _ = path;
9896 }
9897 Ok(())
9898}
9899
9900struct WaveCtx<'a> {
9907 run: &'a str,
9910 node: &'a str,
9912 prompts: &'a Prompts,
9913 cache: Option<&'a Path>,
9915 round: Option<usize>,
9918 carry_seats: bool,
9922 pauses: Option<(&'a Pause, &'a Pause)>,
9925}
9926
9927fn repository_wait(
9929 state: &RunState,
9930 pauses: Option<(&Pause, &Pause)>,
9931) -> crate::repo_lock::WaitPolicy {
9932 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9933 let inherited = crate::repo_lock::WaitPolicy::current();
9934 crate::repo_lock::WaitPolicy::new(
9935 Duration::from_secs(state.config.graph.timeout_resource_wait),
9936 move || {
9937 inherited.cancelled()
9938 || pauses
9939 .as_ref()
9940 .is_some_and(|(a, b)| a.parked() || b.parked())
9941 },
9942 )
9943}
9944
9945fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9946 if error.is::<crate::repo_lock::ResourceBlocked>() {
9947 state.resource_block = Some(crate::run::ResourceBlock {
9948 cause: crate::run::ResourceBlockCause::Other,
9949 node: "repository".into(),
9950 kind: if error.to_string().contains("disk pressure") {
9951 "disk pressure"
9952 } else {
9953 "repository coordination"
9954 }
9955 .into(),
9956 detail: error.to_string(),
9957 at: Timestamp::now(),
9958 });
9959 state.event("resource", error.to_string());
9960 }
9961}
9962
9963async fn coordinate_operation(
9964 state: &mut RunState,
9965 cwd: &Path,
9966 wait: &crate::repo_lock::WaitPolicy,
9967) -> Result<Option<crate::repo_lock::SeatLease>> {
9968 let result =
9969 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9970 .await;
9971 if let Err(e) = &result {
9972 record_repository_block(state, e);
9973 }
9974 result
9975}
9976
9977async fn run_one(
9979 job: SeatJob,
9980 sem: Arc<Semaphore>,
9981 ctx: &WaveCtx<'_>,
9982 state: &mut RunState,
9983 attempt: usize,
9984) -> (SeatState, AgentOutcome) {
9985 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9986 .await
9987 .pop()
9988 .expect("one job in, one result out");
9989 (seat, out)
9990}
9991
9992async fn wave(
9998 jobs: Vec<SeatJob>,
9999 sem: Arc<Semaphore>,
10000 ctx: &WaveCtx<'_>,
10001 state: &mut RunState,
10002 attempt: usize,
10003) -> Vec<(usize, SeatState, AgentOutcome)> {
10004 let WaveCtx {
10005 run,
10006 node,
10007 prompts,
10008 cache,
10009 round,
10010 carry_seats: _,
10011 pauses,
10012 } = *ctx;
10013 for job in &jobs {
10014 state.seat_started(node, &job.seat.key, job.timeout, attempt);
10015 }
10016 if let Err(e) = state.save() {
10017 tracing::warn!("could not persist in-progress seats: {e:#}");
10022 }
10023 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10039 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10043 let cache_guard = if let Some(cache_dir) = cache {
10044 if jobs_had_a_writer {
10045 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10046 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10047 match acquire_cache_lease(
10048 state,
10049 cache_dir,
10050 &owner,
10051 resource_limit,
10052 node,
10053 node,
10054 &cancel,
10055 )
10056 .await
10057 {
10058 Ok(Some(g)) => Some(g),
10059 Ok(None) => {
10062 if let Some(block) = &mut state.resource_block {
10063 block.cause = crate::run::ResourceBlockCause::Cancelled {
10064 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10065 };
10066 }
10067 return block_wave(
10068 jobs,
10069 state,
10070 node,
10071 attempt,
10072 "parked while waiting for the shared build cache",
10073 );
10074 }
10075 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10078 Err(e) => {
10079 let why = format!("{e:#}");
10083 state.event(
10084 "resource",
10085 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10086 );
10087 return block_wave(jobs, state, node, attempt, &why);
10088 }
10089 }
10090 } else {
10091 None
10092 }
10093 } else {
10094 None
10095 };
10096 state.seats_spawning(node, attempt);
10099 if let Err(e) = state.save() {
10100 tracing::warn!("could not persist the start of a wave: {e:#}");
10101 }
10102 let slot_limit = resource_limit
10106 + jobs
10107 .iter()
10108 .map(|j| j.timeout)
10109 .max()
10110 .unwrap_or_default()
10111 .saturating_mul(jobs.len() as u32);
10112 let mut set = tokio::task::JoinSet::new();
10113 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10116 let overlay = prompts.overlay(node);
10117 let repository_wait = repository_wait(state, pauses);
10118 for (i, mut job) in jobs.into_iter().enumerate() {
10119 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10120 if cache.is_some() {
10121 job.prompt.push('\n');
10122 job.prompt
10123 .push_str(&prompt::build_cache_note(node, job.allow_write));
10124 }
10125 let sem = Arc::clone(&sem);
10126 let run = run.to_owned();
10127 let node = node.to_owned();
10128 let attachments = if node == "implement" {
10131 state.attachments.clone()
10132 } else {
10133 Vec::new()
10134 };
10135 let cache = cache
10146 .filter(|_| job.allow_write && cache_guard.is_some())
10147 .map(Path::to_path_buf);
10148 let spawned_tx = spawned_tx.clone();
10149 let repository_wait = repository_wait.clone();
10150 let coordination = state.config.coordination.clone();
10151 let fetched_base = state.fetched_base.clone();
10152 set.spawn(async move {
10153 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10157 Ok(p) => p,
10158 Err(_) => {
10159 return (
10160 i,
10161 job.seat,
10162 AgentOutcome::ResourceBlocked(format!(
10163 "no parallel slot became free within {}s",
10164 slot_limit.as_secs()
10165 )),
10166 false,
10167 );
10168 }
10169 };
10170 spawned_tx.send(job.seat.key.clone()).ok();
10171 let mut seat = job.seat;
10172 let out = repository_wait
10173 .scope(agent::invoke(
10174 &job.spec,
10175 &mut seat,
10176 &Invocation {
10177 coordination: coordination.as_ref(),
10178 fetched_base: fetched_base.as_ref(),
10179 cwd: &job.cwd,
10180 prompt: &job.prompt,
10181 timeout: job.timeout,
10182 allow_write: job.allow_write,
10183 sessions: job.sessions,
10184 artifacts: &job.artifacts,
10185 stem: &job.stem,
10186 run: &run,
10187 node: &node,
10188 cache_dir: cache.as_deref(),
10189 attachments: &attachments,
10190 writable: &[],
10191 },
10192 ))
10193 .await;
10194 let interrupted = out
10195 .as_ref()
10196 .err()
10197 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10198 let out = match out {
10199 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10200 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10201 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10209 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10210 Ok(o) => AgentOutcome::Failed(format!(
10211 "process exited with no usable output ({})",
10212 crate::notices::exit_status(o.exit_code)
10213 )),
10214 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10215 AgentOutcome::ResourceBlocked(e.to_string())
10216 }
10217 Err(e) => AgentOutcome::Failed(e.to_string()),
10218 };
10219 (i, seat, out, interrupted)
10220 });
10221 }
10222 drop(spawned_tx);
10223 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10224 loop {
10225 let joined = tokio::select! {
10226 Some(key) = spawned_rx.recv() => {
10227 state.seat_spawned(&key);
10229 if let Err(e) = state.save() {
10230 tracing::warn!("could not persist a seat's start: {e:#}");
10231 }
10232 continue;
10233 }
10234 joined = set.join_next() => joined,
10235 };
10236 let Some(joined) = joined else {
10237 break;
10238 };
10239 let (i, seat, out, interrupted) = match joined {
10240 Ok(v) => v,
10241 Err(e) => {
10245 tracing::error!("agent task panicked: {e}");
10246 continue;
10247 }
10248 };
10249 state.seat_finished(&seat.key);
10250 if interrupted
10251 && node == "implement"
10252 && let Some(label) = state
10253 .candidates
10254 .iter()
10255 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10256 .map(|candidate| candidate.label)
10257 {
10258 state.candidate_preservation.insert(
10259 label,
10260 "interrupted implementation awaiting persistence".into(),
10261 );
10262 state.interrupted_implementations.insert(label);
10263 }
10264 if let AgentOutcome::ResourceBlocked(why) = &out
10265 && state.resource_block.is_none()
10266 {
10267 state.resource_block = Some(crate::run::ResourceBlock {
10268 cause: crate::run::ResourceBlockCause::Other,
10269 node: node.to_owned(),
10270 kind: if why.contains("disk pressure") {
10271 "disk pressure"
10272 } else if why.contains("repository coordination") {
10273 "repository coordination"
10274 } else {
10275 "parallel slot"
10276 }
10277 .to_owned(),
10278 detail: why.clone(),
10279 at: Timestamp::now(),
10280 });
10281 let activity = if interrupted {
10282 "agent execution interrupted"
10283 } else {
10284 "no agent was started"
10285 };
10286 state.event("resource", format!("{node}: {activity} - {why}"));
10287 }
10288 record_jobs(state, node, round, &seat.key, &out);
10289 if let Err(e) = state.save() {
10290 tracing::warn!("could not persist a seat's completion: {e:#}");
10291 }
10292 if collected.len() <= i {
10293 collected.resize_with(i + 1, || None);
10294 }
10295 collected[i] = Some((i, seat, out));
10296 }
10297 if state
10303 .active
10304 .values()
10305 .any(|a| a.node == node && a.attempt == attempt)
10306 {
10307 state
10308 .active
10309 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10310 if let Err(e) = state.save() {
10311 tracing::warn!("could not persist the end of a wave: {e:#}");
10312 }
10313 }
10314 if let Some(cache_dir) = cache
10321 && jobs_had_a_writer
10322 {
10323 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10324 }
10325 if let Some(guard) = cache_guard {
10326 guard.release();
10327 }
10328 collected.into_iter().flatten().collect()
10329}
10330
10331fn record_jobs(
10342 state: &mut RunState,
10343 node: &str,
10344 round: Option<usize>,
10345 seat: &str,
10346 out: &AgentOutcome,
10347) {
10348 let commands: &[agent::CommandEvidence] = match out {
10349 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10350 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10351 };
10352 let checked_at = Timestamp::now();
10353 for c in commands {
10354 state.jobs.push(JobRecord {
10355 node: node.to_owned(),
10356 round,
10357 seat: seat.to_owned(),
10358 id: c.id.clone(),
10359 description: c.description.clone(),
10360 checked_at,
10361 status: match c.exit_code {
10362 Some(0) => JobStatus::Completed,
10363 Some(_) => JobStatus::Failed,
10364 None => JobStatus::Unknown,
10365 },
10366 exit_code: c.exit_code,
10367 result_summary: c.result_summary.clone(),
10368 source: c.source.clone(),
10369 });
10370 }
10371}
10372
10373fn round_is_clean(
10394 blocking: usize,
10395 e2e_ok: bool,
10396 answered: usize,
10397 expected: usize,
10398 quota_missing: usize,
10399 policy: IncompleteReviewPolicy,
10400) -> bool {
10401 if blocking != 0 || !e2e_ok {
10402 return false;
10403 }
10404 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10405 return true;
10406 }
10407 answered > 0 && expected - answered <= quota_missing
10408}
10409
10410pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10434 if max_rounds == 0
10435 || reviews.last().is_some_and(|r| {
10436 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10437 })
10438 {
10439 return Some(RunStatus::Gating);
10440 }
10441 let last = reviews.last()?;
10442 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10443 if reviews.len() < max_rounds && !stagnant {
10444 return None;
10445 }
10446 if last.incomplete() && last.blocking == 0 {
10447 return Some(RunStatus::Blocked);
10448 }
10449 if last.e2e_status() == E2eStatus::ResourceBlocked
10450 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10451 {
10452 return None;
10453 }
10454 Some(if last.e2e.iter().all(CommandOutcome::ok) {
10455 RunStatus::Gating
10456 } else {
10457 RunStatus::Blocked
10458 })
10459}
10460
10461pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10476 if nudged {
10477 (full / 4).max(Duration::from_secs(120)).min(full)
10478 } else {
10479 full
10480 }
10481}
10482
10483#[allow(clippy::too_many_arguments)]
10512async fn ask_json_wave<T>(
10513 jobs: Vec<SeatJob>,
10514 sem: Arc<Semaphore>,
10515 retries: usize,
10516 roster: &[AgentSpec],
10517 ctx: &WaveCtx<'_>,
10518 losses: &mut Vec<QuotaLoss>,
10519 state: &mut RunState,
10520 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10521) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10522where
10523 T: serde::de::DeserializeOwned + Send + 'static,
10524{
10525 ask_wave_with(
10526 jobs,
10527 sem,
10528 retries,
10529 roster,
10530 ctx,
10531 losses,
10532 state,
10533 &|text: &str| {
10534 let v = verdict::extract_json::<T>(text)?;
10535 validate(&v)?;
10536 Ok(v)
10537 },
10538 )
10539 .await
10540}
10541
10542#[allow(clippy::too_many_arguments)]
10551async fn ask_wave_with<T>(
10552 jobs: Vec<SeatJob>,
10553 sem: Arc<Semaphore>,
10554 retries: usize,
10555 roster: &[AgentSpec],
10556 ctx: &WaveCtx<'_>,
10557 losses: &mut Vec<QuotaLoss>,
10558 state: &mut RunState,
10559 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10560) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10561where
10562 T: Send + 'static,
10563{
10564 let n = jobs.len();
10565 let originals: Vec<SeatJob> = jobs;
10566 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10567 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10568 let mut nudges: Vec<usize> = vec![0; n];
10575 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10579 let mut tried: Vec<BTreeSet<String>> = specs
10580 .iter()
10581 .map(|s| BTreeSet::from([s.id.clone()]))
10582 .collect();
10583 let starts: Vec<usize> = specs
10584 .iter()
10585 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10586 .collect();
10587 let carry = ctx.carry_seats && !roster.is_empty();
10588 let carried: Vec<BTreeSet<String>> = originals
10591 .iter()
10592 .map(|j| {
10593 state
10594 .seat_history
10595 .get(&j.seat.key)
10596 .filter(|_| carry)
10597 .map(|h| h.failed.clone())
10598 .unwrap_or_default()
10599 })
10600 .collect();
10601 let mut prev: Vec<Option<FailClass>> = originals
10602 .iter()
10603 .map(|j| {
10604 state
10605 .seat_history
10606 .get(&j.seat.key)
10607 .filter(|_| carry)
10608 .and_then(|h| h.last_fail.clone())
10609 })
10610 .collect();
10611 let next_agent =
10612 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10613 let others: BTreeSet<String> = specs
10614 .iter()
10615 .enumerate()
10616 .filter(|(j, _)| *j != i)
10617 .map(|(_, s)| s.id.clone())
10618 .collect();
10619 pick_successor(
10620 roster,
10621 starts[i],
10622 tried,
10623 carry.then(|| &carried[i]),
10624 &others,
10625 )
10626 .cloned()
10627 };
10628 let mut fresh: Vec<Option<String>> = vec![None; n];
10629 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
10630 let mut pending: Vec<usize> = (0..n).collect();
10631
10632 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10637 for round in 0..max_rounds {
10638 if pending.is_empty() {
10639 break;
10640 }
10641 let mut batch = Vec::with_capacity(pending.len());
10642 let mut renudged: Vec<&str> = Vec::new();
10643 for &i in &pending {
10644 let src = &originals[i];
10645 let full: &str = match &src.handover {
10650 Some(h) if specs[i].id != src.spec.id => h,
10651 _ => &src.prompt,
10652 };
10653 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10656 (
10657 full.to_owned(),
10658 src.timeout,
10659 format!("{}-{word}-{}", src.stem, specs[i].id),
10660 )
10661 } else if nudges[i] == 0 {
10662 (full.to_owned(), src.timeout, src.stem.clone())
10663 } else {
10664 renudged.push(src.seat.key.as_str());
10665 let why = done[i]
10666 .as_ref()
10667 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10668 .unwrap_or_else(|| "no parsable answer".to_owned());
10669 let nudge = prompt::nudge(&why);
10670 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10671 let prompt = if nudged {
10672 nudge
10673 } else {
10674 format!("{full}\n\n---\n\n{nudge}")
10675 };
10676 (
10677 prompt,
10678 retry_budget(src.timeout, nudged),
10679 format!("{}-retry{}", src.stem, nudges[i]),
10680 )
10681 };
10682 batch.push(SeatJob {
10683 spec: specs[i].clone(),
10684 seat: seats[i].clone(),
10685 cwd: src.cwd.clone(),
10686 prompt,
10687 timeout,
10688 allow_write: src.allow_write,
10689 sessions: src.sessions,
10690 artifacts: src.artifacts.clone(),
10691 stem,
10692 handover: None,
10693 });
10694 }
10695
10696 if !renudged.is_empty() {
10697 state.event(
10698 ctx.node,
10699 format!("retry {round}: re-asking {}", renudged.join(", ")),
10700 );
10701 }
10702 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10703 let mut still = Vec::new();
10704 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10705 seats[i] = seat;
10706 if let AgentOutcome::ResourceBlocked(why) = &out {
10709 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10710 continue;
10711 }
10712 let class = FailClass::of(&out);
10713 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10716 && nudges[i] < retries
10717 && !roster.is_empty();
10718 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10719 let next = should_hand_over(prev[i].as_ref(), &cur)
10720 .then(|| next_agent(i, &tried[i], &specs))
10721 .flatten();
10722 if carry {
10723 let h = state
10724 .seat_history
10725 .entry(originals[i].seat.key.clone())
10726 .or_default();
10727 h.failed.insert(specs[i].id.clone());
10728 h.last_fail = Some(cur.clone());
10729 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10730 h.last_ok = None;
10731 }
10732 if let Err(e) = state.save() {
10735 tracing::warn!("could not persist a seat's failure history: {e:#}");
10736 }
10737 }
10738 if let Some(next) = next {
10739 record_handover(
10740 state,
10741 ctx.node,
10742 &originals[i].seat.key,
10743 &specs[i].id,
10744 &next.id,
10745 &cur,
10746 &fail_reason(&out),
10747 );
10748 tried[i].insert(next.id.clone());
10749 prev[i] = Some(cur.clone());
10750 seats[i] =
10751 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10752 specs[i] = next;
10753 fresh[i] = Some(cur.stem_word().to_owned());
10754 nudges[i] = 0;
10755 done[i] = Some(Err(anyhow::anyhow!(
10756 "handed over after: {}",
10757 fail_reason(&out)
10758 )));
10759 still.push(i);
10760 continue;
10761 }
10762 }
10763 if carry
10764 && !nudge_first
10765 && let Some(cur) = class.clone()
10766 {
10767 let h = state
10769 .seat_history
10770 .entry(originals[i].seat.key.clone())
10771 .or_default();
10772 h.failed.insert(specs[i].id.clone());
10773 h.last_fail = Some(cur);
10774 }
10775 let parsed = match out {
10776 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10777 AgentOutcome::Quota(o) => {
10778 last_quota[i] = o.quota.clone();
10779 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10780 }
10781 AgentOutcome::Dropped(o) => {
10785 let why = o
10786 .dropped
10787 .as_ref()
10788 .map(|d| d.why.as_str())
10789 .unwrap_or("the CLI ended the stream without delivering its answer");
10790 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10791 }
10792 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10793 Err(anyhow::anyhow!(e))
10794 }
10795 };
10796 let quota = class == Some(FailClass::Quota);
10797 let failed = parsed.is_err();
10798 if !failed && matches!(ctx.node, "judge" | "vote") {
10799 state
10802 .quota
10803 .retain(|loss| loss.seat != originals[i].seat.key);
10804 }
10805 done[i] = Some(parsed);
10806 if carry && !failed {
10807 let h = state
10808 .seat_history
10809 .entry(originals[i].seat.key.clone())
10810 .or_default();
10811 h.failed.remove(&specs[i].id);
10812 h.last_ok = Some(specs[i].id.clone());
10813 h.last_fail = None;
10814 }
10815 let agent_failure = class.is_some()
10823 && !nudge_first
10824 && !roster.is_empty()
10825 && next_agent(i, &tried[i], &specs).is_some();
10826 if failed && !quota && !agent_failure && nudges[i] < retries {
10827 nudges[i] += 1;
10828 still.push(i);
10829 }
10830 }
10831 pending = still;
10832 }
10833
10834 for (i, q) in last_quota.into_iter().enumerate() {
10837 if let Some(quota) = q {
10838 losses.push(QuotaLoss::from_report(
10839 originals[i].seat.key.clone(),
10840 ctx.node,
10841 Some("a),
10842 ));
10843 }
10844 }
10845
10846 if state.status != RunStatus::Stalled
10849 && ctx.node != "deliberate"
10850 && n > 0
10851 && losses.len() == n
10852 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
10853 {
10854 block_machine_outage(state, ctx.node, losses);
10855 state.quota.append(losses);
10856 }
10857
10858 seats
10859 .into_iter()
10860 .zip(done)
10861 .zip(nudges)
10862 .map(|((seat, res), attempts)| {
10863 (
10864 seat,
10865 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10866 attempts,
10867 )
10868 })
10869 .collect()
10870}
10871
10872pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
10875 let mut auth: Vec<_> = losses
10876 .iter()
10877 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
10878 .map(|loss| loss.agent.clone())
10879 .collect();
10880 auth.sort();
10881 auth.dedup();
10882 let (kind, detail) = if auth.is_empty() {
10883 (
10884 "agent usage limit",
10885 "All seats reached their agent usage limits; resume after usage is available"
10886 .to_owned(),
10887 )
10888 } else {
10889 (
10890 "agent authentication",
10891 format!(
10892 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
10893 auth.join(", ")
10894 ),
10895 )
10896 };
10897 state.resource_block = Some(crate::run::ResourceBlock {
10898 node: node.to_owned(),
10899 kind: kind.to_owned(),
10900 cause: crate::run::ResourceBlockCause::Other,
10901 detail: detail.clone(),
10902 at: Timestamp::now(),
10903 });
10904 state.event(node, detail);
10905}
10906
10907fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10909 results
10910 .iter()
10911 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10912}
10913
10914fn block_wave(
10918 jobs: Vec<SeatJob>,
10919 state: &mut RunState,
10920 node: &str,
10921 attempt: usize,
10922 why: &str,
10923) -> Vec<(usize, SeatState, AgentOutcome)> {
10924 state.resource_wait = None;
10925 state
10926 .active
10927 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10928 if let Err(e) = state.save() {
10929 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10930 }
10931 jobs.into_iter()
10932 .enumerate()
10933 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10934 .collect()
10935}
10936
10937async fn acquire_cache_lease(
10950 state: &mut RunState,
10951 cache_dir: &Path,
10952 owner: &crate::cache::Owner,
10953 budget: Duration,
10954 node: &str,
10955 context: &str,
10956 cancel: &(dyn Fn() -> bool + Sync),
10957) -> Result<Option<crate::cache::Guard>> {
10958 let home = crate::run::home();
10959 let started = Instant::now();
10960 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10961 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10962 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10963 Err(e) => {
10964 state.event(
10965 "verify",
10966 format!("{context}: could not check the shared build cache: {e:#}"),
10967 );
10968 if let Err(e2) = state.save() {
10969 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10970 }
10971 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10972 }
10973 };
10974 state.event(
10975 "verify",
10976 format!(
10977 "{context}: waiting for the shared build cache at {} ({})",
10978 cache_dir.display(),
10979 busy.describe()
10980 ),
10981 );
10982 state.resource_wait = Some(crate::run::ResourceWait {
10985 node: context.to_owned(),
10986 kind: "shared build cache".to_owned(),
10987 holder: busy.describe(),
10988 since: Timestamp::now(),
10989 limit_secs: budget.as_secs(),
10990 });
10991 if let Err(e) = state.save() {
10992 tracing::warn!("could not persist a cache wait: {e:#}");
10993 }
10994 let remaining = budget.saturating_sub(started.elapsed());
10995 let waited = crate::cache::wait_for_unless(
10996 &home,
10997 cache_dir,
10998 owner,
10999 remaining,
11000 Duration::from_secs(5),
11001 cancel,
11002 )
11003 .await;
11004 state.resource_wait = None;
11005 let cause = match &waited {
11006 Ok(Some(_)) => None,
11007 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11008 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11009 Some(crate::run::ResourceBlockCause::Timeout {
11010 waited_secs: started.elapsed().as_secs(),
11011 limit_secs: budget.as_secs(),
11012 })
11013 }
11014 Err(_) => Some(crate::run::ResourceBlockCause::Other),
11015 };
11016 if let Some(cause) = cause {
11017 let detail = match &waited {
11018 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11019 Err(error) => format!("{error:#}"),
11020 };
11021 state.resource_block = Some(crate::run::ResourceBlock {
11022 cause,
11023 node: node.to_owned(),
11024 kind: "shared build cache".into(),
11025 detail,
11026 at: Timestamp::now(),
11027 });
11028 state.event(
11029 "resource",
11030 state.resource_block.as_ref().unwrap().description(),
11031 );
11032 }
11033 if let Err(error) = state.save() {
11034 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11035 }
11036 waited
11037}
11038
11039#[derive(Debug)]
11044struct CacheCheckError(String);
11045
11046impl std::fmt::Display for CacheCheckError {
11047 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11048 f.write_str(&self.0)
11049 }
11050}
11051
11052impl std::error::Error for CacheCheckError {}
11053
11054fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11056 state
11057 .resource_block
11058 .get_or_insert_with(|| crate::run::ResourceBlock {
11059 cause: crate::run::ResourceBlockCause::Other,
11060 node: node.to_owned(),
11061 kind: "shared build cache".into(),
11062 detail: format!("{error:#}"),
11063 at: Timestamp::now(),
11064 });
11065 if let Err(error) = state.save() {
11066 tracing::warn!("could not persist cache resource block: {error:#}");
11067 }
11068}
11069
11070#[allow(clippy::too_many_arguments)]
11091async fn with_cache_lease<'s, F, Fut>(
11092 state: &'s mut RunState,
11093 cache_dir: Option<&Path>,
11094 node: &str,
11095 seat: &str,
11096 worktree: &Path,
11097 head: &str,
11098 budget: Duration,
11099 context: &str,
11100 wait: &crate::repo_lock::WaitPolicy,
11101 body: F,
11102) -> Result<(Vec<CommandOutcome>, bool)>
11103where
11104 F: FnOnce(&'s mut RunState, Duration) -> Fut,
11105 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11106{
11107 let started = Instant::now();
11110 let home = crate::run::home();
11111 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11112 let guard = if let Some(cache_dir) = cache_dir {
11113 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11114 match acquire_cache_lease(
11115 state,
11116 cache_dir,
11117 &owner,
11118 resource_limit,
11119 node,
11120 context,
11121 &|| wait.cancelled(),
11122 )
11123 .await
11124 .and_then(|g| {
11125 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11126 }) {
11127 Ok(g) => Some(g),
11128 Err(e) => {
11129 record_cache_error(state, node, &e);
11130 return Ok((
11131 vec![CommandOutcome {
11132 command: "(waiting for the shared build cache)".into(),
11133 code: None,
11134 output_tail: e.to_string(),
11135 duration_ms: started.elapsed().as_millis() as u64,
11136 resource_blocked: true,
11137 }],
11138 false,
11139 ));
11140 }
11141 }
11142 } else {
11143 None
11144 };
11145 let _coordination = coordinate_operation(state, worktree, wait).await?;
11146 if let Some(cache_dir) = cache_dir {
11147 let identity = crate::cache::Identity::new(worktree, head);
11148 if let Err(e) = crate::cache::ensure_fresh(
11149 &home,
11150 cache_dir,
11151 &identity,
11152 &state.config.verify.cargo_manifests(),
11153 ) {
11154 record_cache_error(state, node, &e);
11155 state.event(
11156 "verify",
11157 format!(
11158 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11159 worktree.display(),
11160 short(head)
11161 ),
11162 );
11163 if let Some(guard) = guard {
11164 guard.release();
11165 }
11166 return Ok((
11167 vec![CommandOutcome {
11168 command: "(confirming the shared build cache is fresh)".into(),
11169 code: None,
11170 output_tail: e.to_string(),
11171 duration_ms: started.elapsed().as_millis() as u64,
11172 resource_blocked: true,
11173 }],
11174 false,
11175 ));
11176 }
11177 }
11178 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11179 if !timed_out_pids.is_empty() {
11180 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11181 }
11182 drop(_coordination);
11185 if let Some(guard) = guard {
11186 guard.release();
11187 }
11188 Ok((outcomes, retried))
11189}
11190
11191async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11203 wait_for_pids_with(
11204 pids,
11205 crate::proc::pid_alive,
11206 LEASE_RELEASE_POLL,
11207 LEASE_RELEASE_MAX_WAIT,
11208 )
11209 .await;
11210}
11211
11212async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11218 pids: &[u32],
11219 alive: F,
11220 poll: Duration,
11221 max_wait: Duration,
11222) {
11223 let deadline = Instant::now() + max_wait;
11224 loop {
11225 if pids.iter().all(|&pid| !alive(pid)) {
11226 return;
11227 }
11228 if Instant::now() >= deadline {
11229 return;
11230 }
11231 tokio::time::sleep(poll).await;
11232 }
11233}
11234
11235fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11242 outcomes.iter().any(|o| o.resource_blocked)
11243}
11244
11245pub(crate) enum GateFix {
11247 Retry,
11249 Stop,
11252 Defer,
11255}
11256
11257fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11265 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11266 red.peek().is_some()
11267 && red.all(|o| {
11268 !o.resource_blocked
11269 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11270 && !o.output_tail.trim().is_empty()
11271 })
11272}
11273
11274fn e2e_outcome_label(o: &CommandOutcome) -> String {
11278 if o.ok() {
11279 return "pass".to_owned();
11280 }
11281 let reason = if o.build_failed() {
11282 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11283 } else {
11284 format!("FAIL ({:?})", o.code)
11285 };
11286 format!(
11287 "{reason}\n{}",
11288 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11289 )
11290}
11291
11292async fn run_e2e_with_retry(
11300 state: &mut RunState,
11301 shell: &[String],
11302 commands: &[String],
11303 worktree: &Path,
11304 timeout: Duration,
11305 context: &str,
11306) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11307 let (mut e2e, mut timed_out_pids) = run_commands(
11308 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11309 )
11310 .await;
11311 for o in &e2e {
11312 state.event(
11313 "verify",
11314 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11315 );
11316 }
11317 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11321 if verify_retried {
11322 state.event(
11323 "verify",
11324 format!(
11325 "{context}: verify could not build/link, not a test result — retrying once \
11326 before concluding"
11327 ),
11328 );
11329 let retried = run_commands(
11330 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11331 )
11332 .await;
11333 e2e = retried.0;
11334 timed_out_pids.extend(retried.1);
11337 for o in &e2e {
11338 state.event(
11339 "verify",
11340 format!(
11341 "{context}: retry `{}` -> {}",
11342 o.command,
11343 e2e_outcome_label(o)
11344 ),
11345 );
11346 }
11347 }
11348 (e2e, verify_retried, timed_out_pids)
11349}
11350
11351#[allow(clippy::too_many_arguments)]
11367async fn run_commands(
11368 state: &mut RunState,
11369 node: &str,
11370 task: &str,
11371 attempt: usize,
11372 shell: &[String],
11373 commands: &[String],
11374 cwd: &Path,
11375 timeout: Duration,
11376) -> (Vec<CommandOutcome>, Vec<u32>) {
11377 if commands.is_empty() {
11378 return (Vec::new(), Vec::new());
11383 }
11384 let mut out = Vec::new();
11385 let mut timed_out_pids = Vec::new();
11386 let total = commands.len();
11387 for (idx, command) in commands.iter().enumerate() {
11388 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11389 if let Err(e) = state.save() {
11390 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11391 }
11392 let started = Instant::now();
11393 let mut cmd = tokio::process::Command::new(&shell[0]);
11394 cmd.quiet();
11395 for key in crate::worker::INVOCATION_ENV {
11399 cmd.env_remove(key);
11400 }
11401 cmd.args(&shell[1..])
11402 .arg(command)
11403 .current_dir(cwd)
11404 .stdin(std::process::Stdio::null())
11405 .stdout(std::process::Stdio::piped())
11406 .stderr(std::process::Stdio::piped())
11407 .kill_on_drop(true);
11408 let mut outputs = Vec::new();
11409 let protected = (|| -> Result<()> {
11410 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11411 outputs.push(guard);
11412 }
11413 if let Some(cache) = state.config.cache_dir() {
11414 outputs.push(crate::retention::OutputLease::cache(
11415 &crate::run::home(),
11416 &cache,
11417 false,
11418 )?);
11419 }
11420 Ok(())
11421 })();
11422 if let Err(e) = protected {
11423 out.push(CommandOutcome {
11424 command: command.clone(),
11425 code: None,
11426 output_tail: format!("output ownership unavailable: {e:#}"),
11427 duration_ms: 0,
11428 resource_blocked: true,
11429 });
11430 continue;
11431 }
11432 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11433 let (code, body) = match spawned {
11434 Ok(mut running) => {
11435 let pid = running.child().id();
11436 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11437 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11438 let mut paths = vec![cwd.to_owned()];
11439 if let Some(cache) = state.config.cache_dir() {
11440 paths.push(cache);
11441 }
11442 let mut pressure = None;
11443 let status = tokio::select! {
11444 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11445 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11446 pressure = Some(reason); None
11447 }
11448 };
11449 let (code, failure) = match status {
11450 Some(Ok(Ok(status))) => (
11451 status.code(),
11452 status
11453 .code()
11454 .is_none()
11455 .then(|| format!("command terminated: {status}")),
11456 ),
11457 Some(Ok(Err(error))) => {
11458 (None, Some(format!("failed waiting for command: {error}")))
11459 }
11460 Some(Err(_)) => {
11461 if !running.coordinated()
11465 && let Some(pid) = pid
11466 {
11467 timed_out_pids.push(pid);
11468 }
11469 (
11470 None,
11471 Some(format!("timed out after {}s", timeout.as_secs())),
11472 )
11473 }
11474 None => (None, None),
11475 };
11476 running.finish().await;
11477 if let Some(reason) = pressure {
11478 state.resource_block = Some(crate::run::ResourceBlock {
11479 cause: crate::run::ResourceBlockCause::Other,
11480 node: node.to_owned(),
11481 kind: "disk pressure".to_owned(),
11482 detail: reason.clone(),
11483 at: Timestamp::now(),
11484 });
11485 out.push(CommandOutcome {
11486 command: command.clone(),
11487 code: None,
11488 output_tail: reason,
11489 duration_ms: started.elapsed().as_millis() as u64,
11490 resource_blocked: true,
11491 });
11492 break;
11493 }
11494 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11495 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11496 if let Some(failure) = failure {
11497 body.push('\n');
11498 body.push_str(&failure);
11499 }
11500 (code, body)
11501 }
11502 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11503 };
11504 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11505 if !body.is_empty() {
11506 match crate::run::write_verification_artifact(state, task, &body) {
11510 Ok(name) if summary != body => {
11511 summary = format!("[full output: artifacts/{name}]\n{summary}");
11512 }
11513 Ok(_) => {}
11514 Err(e) => {
11515 let warning = format!("could not save the full {task} output: {e:#}");
11516 tracing::warn!("{warning}");
11517 state.event(node, warning);
11518 if summary != body {
11519 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11520 }
11521 }
11522 }
11523 }
11524 out.push(CommandOutcome {
11525 command: command.clone(),
11526 code,
11527 output_tail: summary,
11528 duration_ms: started.elapsed().as_millis() as u64,
11529 resource_blocked: false,
11530 });
11531 }
11532 state.task_finished(task);
11533 if let Err(e) = state.save() {
11534 tracing::warn!("could not persist the end of {task}: {e:#}");
11535 }
11536 (out, timed_out_pids)
11537}
11538
11539fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11551 let repo = repo.display();
11552 match style {
11553 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11554 MergeStyle::Squash => {
11555 let subject = message
11558 .lines()
11559 .next()
11560 .unwrap_or(branch)
11561 .replace(['\\', '"', '$', '`'], "");
11562 format!(
11563 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11564 )
11565 }
11566 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11567 }
11568}
11569
11570const PR_TITLE_MAX: usize = 240;
11581
11582fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11593 let line = queue::first_line(text)?;
11594 let chars: Vec<char> = line.chars().collect();
11595 let (kept, mark) = if chars.len() <= max {
11596 (line.to_owned(), "")
11597 } else if let Some(end) = sentence_end(&chars, max) {
11598 (chars[..end].iter().collect::<String>(), "")
11599 } else {
11600 let room = max.saturating_sub(3);
11601 let cut = if chars[room].is_whitespace() {
11604 room
11605 } else {
11606 chars[..room]
11607 .iter()
11608 .rposition(|c| c.is_whitespace())
11609 .unwrap_or(room)
11610 };
11611 let head: String = chars[..cut].iter().collect();
11612 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11613 (head.to_owned(), "...")
11614 };
11615 if kept.is_empty() {
11616 return None;
11617 }
11618 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11619 return None;
11620 }
11621 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11622 return None;
11623 }
11624 Some(format!("{kept}{mark}"))
11625}
11626
11627fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11631 const MIN: usize = 20;
11632 for i in MIN..max.min(chars.len()) {
11633 if !matches!(chars[i], '.' | '!' | '?') {
11634 continue;
11635 }
11636 let Some(&next) = chars.get(i + 1) else {
11637 continue;
11638 };
11639 if !next.is_whitespace() {
11640 continue;
11641 }
11642 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11643 if after.is_some_and(|c| c.is_ascii_lowercase()) {
11644 continue;
11645 }
11646 let word: String = chars[..i]
11647 .iter()
11648 .rev()
11649 .take_while(|c| !c.is_whitespace())
11650 .collect::<Vec<_>>()
11651 .into_iter()
11652 .rev()
11653 .collect();
11654 let word = word.to_ascii_lowercase();
11655 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11656 continue;
11657 }
11658 let end = chars[..i]
11659 .iter()
11660 .rposition(|c| !c.is_whitespace())
11661 .map_or(i, |p| p + 1);
11662 return Some(end);
11663 }
11664 None
11665}
11666
11667struct PrMessage {
11672 title: String,
11673 body: String,
11674}
11675
11676impl PrMessage {
11677 fn commit_message(&self) -> String {
11681 format!("{}\n\n{}", self.title, self.body)
11682 }
11683}
11684
11685fn title_marker(line: &str) -> Option<&str> {
11687 let line = line.trim();
11688 let head = line.get(..6)?;
11689 head.eq_ignore_ascii_case("title:")
11690 .then(|| line[6..].trim())
11691}
11692
11693fn summary_title(summary: &str) -> Option<String> {
11698 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11699 let raw = title_marker(first)?;
11700 if raw.is_empty() {
11701 return None;
11702 }
11703 let title = queue::title_from(raw, PR_TITLE_MAX);
11704 let lower = title.to_ascii_lowercase();
11705 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11706 return None;
11707 }
11708 Some(title)
11709}
11710
11711const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11713
11714pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11716
11717pub(crate) fn is_review_run(state: &RunState) -> bool {
11721 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11722}
11723
11724fn review_title(state: &RunState) -> Option<String> {
11729 english_subject(
11730 state.reviewed_commits.as_ref()?.first()?,
11731 state.config.graph.github_english(),
11732 )
11733}
11734
11735fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11738 let raw = raw.trim();
11739 if raw.is_empty() {
11740 return None;
11741 }
11742 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11743 return None;
11744 }
11745 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11746 return None;
11747 }
11748 let title = queue::title_from(raw, PR_TITLE_MAX);
11749 let lower = title.to_ascii_lowercase();
11750 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11751 return None;
11752 }
11753 Some(title)
11754}
11755
11756#[derive(Debug, Clone, PartialEq, Eq)]
11759struct BranchFacts {
11760 commits: Vec<(String, String)>,
11762 stat: String,
11764}
11765
11766const STAT_FILE_LINES: usize = 25;
11768const COMMITS_MAX_CHARS: usize = 20_000;
11770
11771async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11774 let commits = git::commit_log(repo, base, branch).await.ok()?;
11775 if commits.is_empty() {
11776 return None;
11777 }
11778 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11779 let lines: Vec<&str> = stat.lines().collect();
11780 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11781 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11782 let more = format!(" ... {omitted} more file(s)");
11783 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11784 kept.push(&more);
11785 kept.push(lines[lines.len() - 1]);
11786 kept.join("\n")
11787 } else {
11788 lines.join("\n")
11789 };
11790 Some(BranchFacts { commits, stat })
11791}
11792
11793fn markdown_safe(text: &str) -> String {
11796 text.replace("</details>", "</details>")
11797 .replace("\x60\x60\x60", "~~~")
11798}
11799
11800fn neutral_title(state: &RunState, winner: char) -> String {
11801 format!(
11802 "chore: land candidate {} of run {}",
11803 winner.to_ascii_uppercase(),
11804 state.id
11805 )
11806}
11807
11808pub(crate) async fn repair_landing_gate(
11811 state: &mut RunState,
11812 winner: &Candidate,
11813 outcomes: &[CommandOutcome],
11814) -> Result<GateFix> {
11815 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11816 return Ok(GateFix::Stop);
11817 }
11818 let mut runner = Runner {
11819 roles: state.config.resolve_roles()?,
11820 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11821 state: state.clone(),
11822 pause: Pause::new(),
11823 interrupt: Pause::new(),
11824 };
11825 let result = runner.gate_fix_round(winner, outcomes).await;
11826 *state = runner.state;
11827 result
11828}
11829
11830pub(crate) async fn verify_current_head(
11833 state: &mut RunState,
11834 winner: &Candidate,
11835 wait: &crate::repo_lock::WaitPolicy,
11836) -> Result<bool> {
11837 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11838 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11840 state
11841 .verification_history
11842 .push(crate::run::VerificationEvidence {
11843 superseded_by: Some(head.clone()),
11844 archived_at: Some(Timestamp::now()),
11845 fixes_spent: Some(state.gate_fixes.len()),
11846 gate_head: None,
11847 gate_ran: false,
11848 gate: Vec::new(),
11849 e2e: state.head_e2e.take(),
11850 });
11851 }
11852 if let Some(e) = &state.head_e2e
11853 && e.head == head
11854 {
11855 if e.passed {
11856 return Ok(true);
11857 }
11858 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11859 state.status = RunStatus::Blocked;
11860 state.save()?;
11861 return Ok(false);
11862 }
11863 } else if state.reviews.last().is_some_and(|r| {
11864 r.verified_head.as_deref() == Some(head.as_str())
11865 && (r.e2e_status() == E2eStatus::Passed
11866 || (r.e2e_status() == E2eStatus::NotConfigured
11867 && state.config.verify.e2e.is_empty()))
11868 }) || state
11869 .gate_repair_e2e
11870 .as_ref()
11871 .is_some_and(|e| e.head == head && e.passed)
11872 {
11873 return Ok(true);
11874 }
11875 if let Some(e) = &state.gate_repair_e2e
11876 && e.head == head
11877 && !e.passed
11878 && !e.outcomes.is_empty()
11879 && !verify_inconclusive(&e.outcomes)
11880 {
11881 state.status = RunStatus::Blocked;
11882 state.save()?;
11883 return Ok(false);
11884 }
11885 state.head_e2e = Some(crate::run::GateRepairE2e {
11886 head: head.clone(),
11887 passed: false,
11888 outcomes: Vec::new(),
11889 });
11890 state.save()?;
11891 let commands = state.config.verify.e2e.clone();
11892 let shell = state.config.shell();
11893 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11894 let cache_dir = state.config.cache_dir();
11895 let outcomes = if commands.is_empty() {
11896 Vec::new()
11897 } else {
11898 let (outcomes, _) = with_cache_lease(
11899 state,
11900 cache_dir.as_deref(),
11901 "e2e",
11902 "e2e",
11903 &winner.worktree,
11904 &head,
11905 timeout,
11906 "current head e2e",
11907 wait,
11908 |state, budget| async move {
11909 run_e2e_with_retry(
11910 state,
11911 &shell,
11912 &commands,
11913 &winner.worktree,
11914 budget,
11915 "current head e2e",
11916 )
11917 .await
11918 },
11919 )
11920 .await?;
11921 outcomes
11922 };
11923 let unresolved = verify_inconclusive(&outcomes);
11924 let passed = !unresolved
11925 && outcomes.iter().all(CommandOutcome::ok)
11926 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11927 state.head_e2e = Some(crate::run::GateRepairE2e {
11928 head: head.clone(),
11929 passed,
11930 outcomes,
11931 });
11932 if !passed && !unresolved {
11933 state.status = RunStatus::Blocked;
11934 state.event(
11935 "e2e",
11936 format!("verify.e2e failed on head {head}; nothing published"),
11937 );
11938 }
11939 state.save()?;
11940 Ok(passed)
11941}
11942
11943pub(crate) async fn check_gate_repair_e2e(
11946 state: &mut RunState,
11947 winner: &Candidate,
11948 wait: &crate::repo_lock::WaitPolicy,
11949) -> Result<GateFix> {
11950 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11951 return Ok(GateFix::Retry);
11952 }
11953 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11954 if state
11955 .gate_repair_e2e
11956 .as_ref()
11957 .is_some_and(|r| r.head == head && r.passed)
11958 {
11959 return Ok(GateFix::Retry);
11960 }
11961 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11962 head: head.clone(),
11963 passed: false,
11964 outcomes: Vec::new(),
11965 });
11966 state.save()?;
11967 let commands = state.config.verify.e2e.clone();
11968 let outcomes = if commands.is_empty() {
11969 Vec::new()
11970 } else {
11971 let shell = state.config.shell();
11972 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11973 let cache_dir = state.config.cache_dir();
11974 let (outcomes, _) = with_cache_lease(
11975 state,
11976 cache_dir.as_deref(),
11977 "e2e",
11978 "e2e",
11979 &winner.worktree,
11980 &head,
11981 timeout,
11982 "gate repair e2e",
11983 wait,
11984 |state, budget| async move {
11985 run_e2e_with_retry(
11986 state,
11987 &shell,
11988 &commands,
11989 &winner.worktree,
11990 budget,
11991 "gate repair e2e",
11992 )
11993 .await
11994 },
11995 )
11996 .await?;
11997 outcomes
11998 };
11999 let deferred = verify_inconclusive(&outcomes);
12000 let passed = outcomes.iter().all(CommandOutcome::ok)
12001 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12002 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12003 head,
12004 passed,
12005 outcomes,
12006 });
12007 state.save()?;
12008 if deferred {
12009 Ok(GateFix::Defer)
12010 } else if !passed {
12011 state.event(
12012 "gate",
12013 "verify.e2e failed after gate repair; nothing published",
12014 );
12015 Ok(GateFix::Stop)
12016 } else {
12017 Ok(GateFix::Retry)
12018 }
12019}
12020
12021pub(crate) async fn run_landing_gate(
12023 state: &mut RunState,
12024 winner: &Candidate,
12025 wait: &crate::repo_lock::WaitPolicy,
12026) -> Result<Vec<CommandOutcome>> {
12027 let shell = state.config.shell();
12028 let gate_commands = state.config.verify.gate.clone();
12029 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12030 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12031 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12032 Vec::new()
12033 } else {
12034 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12035 let cache_dir = state.config.cache_dir();
12036 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12037 let (outcomes, _) = with_cache_lease(
12038 state,
12039 cache_dir.as_deref(),
12040 "gate",
12041 "gate",
12042 &winner.worktree,
12043 &head,
12044 timeout,
12045 "final gate",
12046 wait,
12047 |state, budget| {
12048 let shell = shell.clone();
12049 let gate_commands = gate_commands.clone();
12050 let winner = winner.clone();
12051 async move {
12052 Runner::run_pre_gate(state, &winner).await;
12053 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12055 let (outcomes, timed_out_pids) = run_commands(
12056 state,
12057 "gate",
12058 "gate",
12059 0,
12060 &shell,
12061 &gate_commands,
12062 &winner.worktree,
12063 budget,
12064 )
12065 .await;
12066 (outcomes, false, timed_out_pids)
12067 }
12068 },
12069 )
12070 .await?;
12071 outcomes
12072 };
12073 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12074 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12075 let mut outcomes = outcomes;
12076 outcomes.push(CommandOutcome {
12077 command: "gate head stability".into(), code: None,
12078 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12079 duration_ms: 0, resource_blocked: false,
12080 });
12081 return Ok(outcomes);
12082 }
12083 if outcomes.is_empty() {
12084 state.event("gate", "no gate commands configured; not tested");
12089 }
12090 for o in &outcomes {
12091 state.event(
12092 "gate",
12093 format!(
12094 "`{}` -> {}",
12095 o.command,
12096 if o.ok() {
12097 "pass".to_owned()
12098 } else {
12099 format!(
12100 "FAIL ({:?})\n{}",
12101 o.code,
12102 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12103 )
12104 }
12105 ),
12106 );
12107 }
12108 Ok(outcomes)
12109}
12110
12111pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12117 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12118 ""
12119 } else {
12120 pr_title
12121 }
12122}
12123
12124pub fn landing_subject_source(state: &RunState) -> String {
12128 if is_review_run(state) {
12129 let winner = state.candidates.first().map_or('A', |c| c.label);
12130 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12131 }
12132 state.instruction.clone()
12133}
12134
12135fn summary_without_title(summary: &str) -> String {
12138 let mut lines = summary.trim().lines().peekable();
12139 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12140 lines.next();
12141 }
12142 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12143}
12144
12145#[cfg(test)]
12157fn pr_message(state: &RunState, winner: char) -> PrMessage {
12158 pr_message_with(state, winner, None)
12159}
12160
12161#[cfg(test)]
12164fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12165 let raw = pr_message_raw(
12166 state,
12167 winner,
12168 facts,
12169 crate::pr_outcome::FinalHead::Unreported,
12170 ChangeNote::NotAsked,
12171 );
12172 let id = crate::scrub::Identity::current();
12173 PrMessage {
12174 title: crate::scrub::scrub(&raw.title, &id),
12175 body: crate::scrub::scrub(&raw.body, &id),
12176 }
12177}
12178
12179#[derive(Debug, Clone, Copy)]
12181enum ChangeNote<'a> {
12182 NotAsked,
12184 Summary(&'a crate::run::ChangeSummary),
12186 Unavailable(&'a str),
12188}
12189
12190fn pr_message_raw(
12191 state: &RunState,
12192 winner: char,
12193 facts: Option<&BranchFacts>,
12194 head: crate::pr_outcome::FinalHead<'_>,
12195 change: ChangeNote<'_>,
12196) -> PrMessage {
12197 if !matches!(change, ChangeNote::NotAsked) {
12198 let (title, mut body) = match change {
12199 ChangeNote::Summary(c) => (
12200 c.title.clone(),
12201 format!("## Summary\n\n{}\n\n", c.body.trim()),
12202 ),
12203 ChangeNote::Unavailable(why) => (
12204 neutral_title(state, winner),
12205 format!(
12206 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12207 ),
12208 ),
12209 ChangeNote::NotAsked => unreachable!(),
12210 };
12211 body.push_str(&public_pr_evidence(state, head));
12212 body.push_str(&format!(
12213 "\n---\nmagi:run/{} magi:candidate-{}\n",
12214 state.id,
12215 winner.to_ascii_lowercase()
12216 ));
12217 return PrMessage { title, body };
12218 }
12219 pr_preview(state, winner, facts, head)
12220}
12221
12222fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12225 let mut body = crate::pr_outcome::review_rounds(state);
12226 body.push_str(&crate::pr_outcome::final_verification(state, head));
12227 let open = state.last_round_findings();
12228 if !open.is_empty() {
12229 body.push_str("## Open review findings\n\n");
12230 for f in open {
12231 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12232 }
12233 body.push('\n');
12234 }
12235 if let Some(c) = state.change_summary.as_ref()
12236 && head == crate::pr_outcome::FinalHead::At(&c.head)
12237 {
12238 body.push_str(&format!(
12239 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12240 c.base, c.head
12241 ));
12242 }
12243 body
12244}
12245
12246fn pr_preview(
12248 state: &RunState,
12249 winner: char,
12250 facts: Option<&BranchFacts>,
12251 head: crate::pr_outcome::FinalHead<'_>,
12252) -> PrMessage {
12253 let summary = state
12254 .candidates
12255 .iter()
12256 .find(|c| c.label == winner)
12257 .map(|c| c.summary.as_str())
12258 .unwrap_or_default();
12259 let review = is_review_run(state);
12262 let title = if review {
12263 facts
12264 .and_then(|f| {
12265 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12266 })
12267 .or_else(|| review_title(state))
12268 .or_else(|| {
12269 state
12270 .candidates
12271 .iter()
12272 .find(|c| c.label == winner)
12273 .filter(|c| !c.branch.starts_with("magi/"))
12274 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12275 })
12276 .unwrap_or_else(|| neutral_title(state, winner))
12277 } else {
12278 summary_title(summary).unwrap_or_else(|| {
12279 english_title(
12280 &state.instruction,
12281 PR_TITLE_MAX,
12282 state.config.graph.github_english(),
12283 )
12284 .unwrap_or_else(|| neutral_title(state, winner))
12285 })
12286 };
12287
12288 let mut body = String::new();
12289 let what = summary_without_title(summary);
12290 if !what.is_empty() {
12291 body.push_str("## Summary\n\n");
12292 body.push_str(&what);
12293 body.push_str("\n\n");
12294 }
12295
12296 body.push_str(&crate::pr_outcome::review_rounds(state));
12298 body.push_str(&crate::pr_outcome::final_verification(state, head));
12299 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12300
12301 let open = state.open_findings();
12302 if !open.is_empty() {
12303 body.push_str("## Open review findings\n\n");
12304 for f in &open {
12305 body.push_str(&format!(
12306 "- `{}` [{:?}] {}\n",
12307 f.id, f.severity, "Context retained in local review records."
12308 ));
12309 }
12310 body.push('\n');
12311 }
12312
12313 if let Some(fix) = fix
12314 && !fix.rejected.is_empty()
12315 {
12316 body.push_str("## Declined by the fixer\n\n");
12317 for r in &fix.rejected {
12318 body.push_str(&format!(
12319 "- `{}`: {}\n",
12320 r.id, "Reason retained in local review records."
12321 ));
12322 }
12323 body.push('\n');
12324 }
12325
12326 if review {
12327 body.push_str("## Commits under review\n\n");
12329 if let Some(facts) = facts {
12330 let mut left = COMMITS_MAX_CHARS;
12331 for (i, (subject, text)) in facts.commits.iter().enumerate() {
12332 let mut entry = format!("- {}\n", markdown_safe(subject));
12333 for l in markdown_safe(text).lines() {
12334 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
12335 }
12336 if left == 0 {
12337 body.push_str(&format!(
12338 "- ... {} more commit(s)\n",
12339 facts.commits.len() - i
12340 ));
12341 break;
12342 }
12343 if entry.len() > left {
12344 let mut end = left;
12347 while !entry.is_char_boundary(end) {
12348 end -= 1;
12349 }
12350 entry.truncate(end);
12351 entry.push_str("\n ... (truncated)\n");
12352 left = 0;
12353 } else {
12354 left -= entry.len();
12355 }
12356 body.push_str(&entry);
12357 }
12358 if !facts.stat.trim().is_empty() {
12359 body.push_str(&format!(
12360 "\n## Diff stat\n\n```\n{}\n```\n",
12361 markdown_safe(facts.stat.trim())
12362 ));
12363 }
12364 } else {
12365 match &state.reviewed_commits {
12366 Some(subjects) => {
12367 for s in subjects {
12368 body.push_str(&format!("- {}\n", s.trim()));
12369 }
12370 }
12371 None => {
12372 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12375 body.push_str(rest.trim());
12376 body.push('\n');
12377 }
12378 }
12379 }
12380 } else {
12381 let task = state.instruction.trim();
12382 let task = if task.is_empty() {
12383 "(empty task)"
12384 } else {
12385 task
12386 };
12387 body.push_str(&format!(
12388 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12389 task.replace("</details>", "</details>")
12390 ));
12391 }
12392
12393 body.push_str(&format!(
12394 "\n---\nmagi:run/{} magi:candidate-{}\n",
12395 state.id,
12396 winner.to_ascii_lowercase()
12397 ));
12398
12399 PrMessage { title, body }
12400}
12401
12402fn seeded_instruction(state: &RunState) -> String {
12405 let mut inputs = Vec::new();
12406 if let Some(start) = &state.sources.start_from {
12407 inputs.push(format!(
12408 "Start from `{}` at `{}` (including ancestry).",
12409 start.token, start.sha
12410 ));
12411 }
12412 for pick in &state.sources.pick_commits {
12413 inputs.push(format!(
12414 "Pick `{}` at `{}` (single commit).",
12415 pick.token, pick.sha
12416 ));
12417 }
12418 if inputs.is_empty() {
12419 state.instruction.clone()
12420 } else {
12421 format!(
12422 "{}\n\n# Explicit source inputs already prepared\n\n{}",
12423 state.instruction,
12424 inputs.join("\n")
12425 )
12426 }
12427}
12428
12429async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12433 let base = &state.base_branch;
12434 let mut against = base.clone();
12435 if mode == MergeMode::Pr {
12436 let remote = &state.config.merge.remote;
12440 let tracking = format!("{remote}/{base}");
12441 let fetched = git::fetch(repo, remote, base).await;
12442 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12443 against = tracking;
12444 } else {
12445 return false;
12446 }
12447 }
12448 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12449}
12450
12451fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12454 let mut detail = format!(
12455 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12456 nothing to open a pull request for"
12457 );
12458 detail.push_str(
12459 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12460 );
12461 detail
12462}
12463
12464#[derive(Debug, PartialEq, Eq)]
12467enum PrPlan {
12468 Create,
12469 Adopt { url: String, title: String },
12470 Stop(String),
12471}
12472
12473fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12476 match found {
12477 Ok(land::OpenPr::None) => PrPlan::Create,
12478 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12479 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12480 "several open pull requests exist for this branch, not picking one: {}",
12481 urls.join(" ")
12482 )),
12483 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12484 }
12485}
12486
12487pub(crate) async fn gh_pr_create(
12489 cwd: &Path,
12490 base: &str,
12491 head: &str,
12492 title: &str,
12493 body: &str,
12494) -> Result<String> {
12495 let out = tokio::process::Command::new("gh")
12496 .args([
12497 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12498 ])
12499 .current_dir(cwd)
12500 .quiet()
12501 .stdin(std::process::Stdio::null())
12502 .output()
12503 .await
12504 .context("spawn gh")?;
12505 if out.status.success() {
12506 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12507 } else {
12508 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12509 }
12510}
12511
12512async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12514 match std::fs::symlink_metadata(path) {
12515 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12516 git::worktree_remove_clean(repo, path).await?;
12519 return Ok(());
12520 }
12521 Err(e) => return Err(e.into()),
12522 Ok(_) => {}
12523 }
12524 if !git::is_clean(path).await? {
12525 bail!(
12526 "worktree has uncommitted work; refusing removal: {}",
12527 path.display()
12528 );
12529 }
12530 if !git::worktree_remove_clean(repo, path).await? {
12531 bail!("worktree removal refused: {}", path.display());
12532 }
12533 Ok(())
12534}
12535
12536pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12545 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12546 bail!("refusing fold: {reason}");
12547 }
12548 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12549 bail!("candidate preservation incomplete; refusing cleanup");
12550 }
12551 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12555 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12556 let _driver = crate::driver::maintenance(home, &state.id)?;
12557 let mut output_guards = Vec::new();
12558 for entry in std::fs::read_dir(state.worktree_root())
12559 .into_iter()
12560 .flatten()
12561 .flatten()
12562 {
12563 if entry.path().join(".git").is_file() {
12564 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12565 }
12566 }
12567 let repo = state.repo.clone();
12568 let wait = repository_wait(state, None);
12571 let _coordination = match coordinate_operation(state, &repo, &wait).await {
12572 Ok(lease) => lease,
12573 Err(e) => {
12574 state.save_under(home)?;
12575 return Err(e);
12576 }
12577 };
12578 let _source_outputs = if state.status == RunStatus::Merged
12581 && state.merge.as_ref().is_some_and(|merge| merge.ok)
12582 {
12583 verify_merged_fold_sources(state, drop_winner, home).await?
12584 } else {
12585 Vec::new()
12586 };
12587 let root = state.worktree_root();
12588 let winner = state.tally.as_ref().map(|t| t.winner);
12589 let mut removed = Vec::new();
12590
12591 for i in 0..state.candidates.len() {
12592 let c = state.candidates[i].clone();
12593 let is_winner = Some(c.label) == winner;
12594 if is_winner && !drop_winner {
12595 continue;
12596 }
12597 let existed = c.worktree.exists();
12598 remove_preserved_worktree(&repo, &c.worktree).await?;
12601 if existed {
12602 removed.push(c.worktree.to_string_lossy().into_owned());
12603 }
12604 let handed_over = state.released_branches.contains(&c.branch);
12607 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12608 if !git::branch_delete(&repo, &c.branch).await? {
12609 bail!("candidate branch removal refused");
12610 }
12611 removed.push(c.branch.clone());
12612 }
12613 state.candidates[i].folded = true;
12614 }
12615
12616 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12617 let path = name.path();
12618 let keep = !drop_winner
12619 && winner.is_some_and(|w| {
12620 path.file_name()
12621 .is_some_and(|n| n == format!("cand-{w}").as_str())
12622 });
12623 if keep {
12624 continue;
12625 }
12626 remove_preserved_worktree(&repo, &path).await?;
12627 removed.push(path.to_string_lossy().into_owned());
12628 }
12629
12630 remove_if_empty(&root);
12639
12640 if state.enabled_worktree_config && drop_winner {
12641 git::release_worktree_config(&repo).await.ok();
12645 state.enabled_worktree_config = false;
12646 }
12647 state.save_under(home)?;
12648 Ok(removed)
12649}
12650
12651async fn verify_merged_fold_sources(
12652 state: &RunState,
12653 drop_winner: bool,
12654 home: &Path,
12655) -> Result<Vec<std::fs::File>> {
12656 let winner = state.tally.as_ref().map(|t| t.winner);
12657 let mut heads = BTreeSet::new();
12658 for candidate in &state.candidates {
12659 if Some(candidate.label) == winner && !drop_winner {
12660 continue;
12661 }
12662 anyhow::ensure!(
12663 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12664 "candidate recovery source is outside the inventoried run root: {}",
12665 candidate.worktree.display()
12666 );
12667 match std::fs::symlink_metadata(&candidate.worktree) {
12668 Ok(_) => {}
12669 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12670 Err(error) => return Err(error.into()),
12671 }
12672 if !state.released_branches.contains(&candidate.branch)
12673 && git::branch_exists(&state.repo, &candidate.branch).await?
12674 {
12675 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12676 }
12677 }
12678 let mut cargo_guards = Vec::new();
12679 let entries = match std::fs::read_dir(state.worktree_root()) {
12680 Ok(entries) => Some(entries),
12681 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12682 Err(error) => return Err(error.into()),
12683 };
12684 for entry in entries.into_iter().flatten() {
12685 let path = entry?.path();
12686 let metadata = std::fs::symlink_metadata(&path)?;
12687 anyhow::ensure!(
12688 metadata.is_dir() && !metadata.file_type().is_symlink(),
12689 "non-directory or symlink recovery source retained: {}",
12690 path.display()
12691 );
12692 if !drop_winner
12693 && winner.is_some_and(|w| {
12694 path.file_name()
12695 .is_some_and(|name| name == format!("cand-{w}").as_str())
12696 })
12697 {
12698 continue;
12699 }
12700 anyhow::ensure!(
12701 path.join(".git").is_file(),
12702 "unregistered recovery source retained: {}",
12703 path.display()
12704 );
12705 anyhow::ensure!(
12706 git::git(
12707 &path,
12708 &[
12709 "status",
12710 "--porcelain",
12711 "--untracked-files=all",
12712 "--ignored",
12713 "--",
12714 ".",
12715 ":(exclude)target"
12716 ]
12717 )
12718 .await?
12719 .is_empty(),
12720 "source changes or ignored recovery files retained: {}",
12721 path.display()
12722 );
12723 match std::fs::symlink_metadata(path.join("target")) {
12724 Ok(_) => {
12725 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12726 }
12727 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12728 Err(error) => return Err(error.into()),
12729 }
12730 for checkpoint in [
12731 "rebase-merge",
12732 "rebase-apply",
12733 "sequencer",
12734 "CHERRY_PICK_HEAD",
12735 "MERGE_HEAD",
12736 "REBASE_HEAD",
12737 "REVERT_HEAD",
12738 "BISECT_LOG",
12739 "AUTO_MERGE",
12740 "MERGE_MSG",
12741 ] {
12742 let location = git::git(
12743 &path,
12744 &[
12745 "rev-parse",
12746 "--path-format=absolute",
12747 "--git-path",
12748 checkpoint,
12749 ],
12750 )
12751 .await?;
12752 anyhow::ensure!(
12753 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12754 if e.kind() == std::io::ErrorKind::NotFound),
12755 "Git recovery checkpoint retained: {checkpoint}"
12756 );
12757 }
12758 heads.insert(git::rev_parse(&path, "HEAD").await?);
12759 }
12760 crate::fold_evidence::preserve(state, home, &heads).await?;
12761
12762 Ok(cargo_guards)
12763}
12764
12765fn remove_if_empty(dir: &Path) {
12776 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12777 std::fs::remove_dir(dir).ok();
12778 }
12779}
12780
12781pub fn worst_open(state: &RunState) -> Option<Severity> {
12783 state
12784 .reviews
12785 .last()?
12786 .reviews
12787 .iter()
12788 .flat_map(|r| r.findings.iter())
12789 .map(|f| f.severity)
12790 .max()
12791}
12792
12793async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12797 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12798 git::acquire_worktree_config(repo).await?;
12799 state.enabled_worktree_config = true;
12800 }
12801 Ok(())
12802}
12803
12804#[cfg(test)]
12805mod tests {
12806 #[tokio::test]
12807 async fn launcher_base_fetch_requires_remote_success() {
12808 let tmp = tempfile::tempdir().unwrap();
12809 let repo = tmp.path().join("repo");
12810 let remote = tmp.path().join("remote");
12811 std::fs::create_dir(&repo).unwrap();
12812 std::fs::create_dir(&remote).unwrap();
12813 git::git(&remote, &["init", "--bare", "-b", "main"])
12814 .await
12815 .unwrap();
12816 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12817 git::git(
12818 &repo,
12819 &[
12820 "-c",
12821 "user.name=Fixture",
12822 "-c",
12823 "user.email=fixture@example.invalid",
12824 "commit",
12825 "--allow-empty",
12826 "-m",
12827 "fixture",
12828 ],
12829 )
12830 .await
12831 .unwrap();
12832 git::git(
12833 &repo,
12834 &["remote", "add", "origin", remote.to_str().unwrap()],
12835 )
12836 .await
12837 .unwrap();
12838 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12839 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12840 assert_eq!(
12841 resolve_base(&repo, "main", "origin", None)
12842 .await
12843 .unwrap()
12844 .commit,
12845 head
12846 );
12847 #[cfg(unix)]
12848 {
12849 let policy: crate::config::Coordination = toml::from_str(
12850 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12851 ).unwrap();
12852 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12853 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12854 let held = crate::repo_lock::SeatLease::operation_with(
12855 &repo,
12856 Some(&policy),
12857 &crate::repo_lock::WaitPolicy::default(),
12858 )
12859 .await
12860 .unwrap();
12861 assert!(held.is_some());
12862 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12863 tokio::pin!(fetch);
12864 assert!(
12865 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12866 .await
12867 .is_err()
12868 );
12869 assert!(
12870 !repo.join(".git/FETCH_HEAD").exists(),
12871 "no fetch before admission"
12872 );
12873 drop(held);
12874 let fetched = fetch.await.unwrap();
12875 assert_eq!(fetched.commit, head);
12876 assert_eq!(fetched.remote, "origin");
12877 assert_eq!(fetched.branch, "main");
12878 }
12879 git::git(
12881 &repo,
12882 &[
12883 "remote",
12884 "set-url",
12885 "origin",
12886 tmp.path().join("missing").to_str().unwrap(),
12887 ],
12888 )
12889 .await
12890 .unwrap();
12891 let error = resolve_base(&repo, "main", "origin", None)
12892 .await
12893 .unwrap_err()
12894 .to_string();
12895 assert!(error.contains("cannot read origin/main"));
12896 assert!(error.contains("refusing to branch off"));
12897 }
12898
12899 #[test]
12900 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12901 let leaked = vec!["chore(deps): update a crate".to_owned()];
12902 let own = "fix(daemon): apply a chosen action";
12903 assert!(should_retitle("", own, &leaked));
12904 assert!(should_retitle(
12905 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12906 own,
12907 &leaked
12908 ));
12909 assert!(should_retitle(
12910 "chore: land candidate A of run 1",
12911 own,
12912 &leaked
12913 ));
12914 assert!(should_retitle(
12915 "magi: candidate A (uncommitted work)",
12916 own,
12917 &leaked
12918 ));
12919 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12920 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12921 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12922 }
12923
12924 #[test]
12925 fn pr_merge_plan_creates_adopts_or_stops() {
12926 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12927 assert_eq!(
12928 pr_merge_plan(Ok(land::OpenPr::One {
12929 url: "u".into(),
12930 title: "t".into()
12931 })),
12932 PrPlan::Adopt {
12933 url: "u".into(),
12934 title: "t".into()
12935 }
12936 );
12937 let PrPlan::Stop(many) =
12938 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12939 else {
12940 panic!("many must stop");
12941 };
12942 assert!(many.contains('a') && many.contains('b'));
12943 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12944 panic!("a failed lookup must stop");
12945 };
12946 assert!(err.contains("bad token"));
12947 }
12948
12949 use super::*;
12950 use crate::run::GateStatus;
12951 use std::collections::BTreeMap;
12952 use std::time::Duration;
12953
12954 fn conductor() -> AgentSpec {
12955 AgentSpec {
12956 id: "conductor".to_owned(),
12957 kind: crate::config::AgentKind::Command,
12958 model: None,
12959 command: vec!["true".to_owned()],
12960 extra_args: Vec::new(),
12961 env: BTreeMap::new(),
12962 prompt_delivery: None,
12963 max_turn_secs: None,
12964 }
12965 }
12966
12967 fn spec(id: &str) -> AgentSpec {
12968 AgentSpec {
12969 id: id.to_owned(),
12970 kind: crate::config::AgentKind::Command,
12971 model: None,
12972 command: vec!["true".to_owned()],
12973 extra_args: Vec::new(),
12974 env: BTreeMap::new(),
12975 prompt_delivery: None,
12976 max_turn_secs: None,
12977 }
12978 }
12979
12980 fn ids(xs: &[&str]) -> BTreeSet<String> {
12981 xs.iter().map(|s| (*s).to_owned()).collect()
12982 }
12983
12984 #[test]
12985 fn pick_successor_skips_an_agent_another_seat_holds() {
12986 let roster = [spec("a"), spec("b"), spec("c")];
12988 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12989 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12990 }
12991
12992 #[test]
12993 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12994 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12996 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12997 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12998 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12999 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13000 }
13001
13002 #[test]
13003 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13004 let roster = [spec("a"), spec("b")];
13005 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13006 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13007 let carried = ids(&["b"]);
13008 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13009 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13010 }
13011
13012 #[test]
13013 fn pick_successor_returns_none_without_any_untried_successor() {
13014 let roster = [spec("a"), spec("b")];
13015 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13016 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13017 let carried = ids(&["a"]);
13018 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13019 }
13020
13021 #[test]
13022 fn pick_successor_rescue_avoids_another_seats_occupant() {
13023 let roster = [spec("a"), spec("b"), spec("c")];
13025 let carried = ids(&["b", "c"]);
13026 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13027 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13028 }
13029
13030 #[test]
13031 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13032 let roster = [spec("a"), spec("b"), spec("c")];
13033 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13034 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13035 }
13036
13037 #[test]
13038 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13039 let roster = [spec("a"), spec("b"), spec("c")];
13040 let failed = ids(&["a", "b", "c"]);
13041 let mut tried = ids(&["b"]);
13043 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13044 assert_eq!(first.id, "a");
13045 tried.insert(first.id.clone());
13046 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13047 assert_eq!(second.id, "c");
13048 tried.insert(second.id.clone());
13049 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13050 }
13051
13052 #[test]
13053 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13054 let roster = [spec("a"), spec("b")];
13055 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13056 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13057 }
13058
13059 #[test]
13060 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13061 let roster = [spec("a"), spec("b"), spec("c")];
13062 let h = SeatHistory {
13063 failed: ids(&["a"]),
13064 last_ok: Some("b".to_owned()),
13065 last_fail: None,
13066 };
13067 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13068 let gone = SeatHistory {
13070 last_ok: Some("zzz".to_owned()),
13071 ..h.clone()
13072 };
13073 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13074 let failed = SeatHistory {
13075 failed: ids(&["a", "b"]),
13076 last_ok: Some("b".to_owned()),
13077 last_fail: None,
13078 };
13079 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13080 }
13081
13082 #[test]
13083 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13084 let first = SeatState::new("review-1", "alpha", 7);
13085 let next = handover_seat("review-1", "gamma", 7);
13086 assert_ne!(first.claude_session, next.claude_session);
13087 }
13088
13089 #[test]
13090 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13091 let mut state = state_with_summary("x", "y");
13092 let a = handover_seat("review-1", "beta", state.next_seat_seed());
13093 let b = handover_seat("review-1", "beta", state.next_seat_seed());
13094 assert_ne!(a.claude_session, b.claude_session);
13095 }
13096
13097 #[test]
13098 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13099 let roster = [spec("a"), spec("b")];
13100 let h = SeatHistory {
13101 failed: ids(&["a", "b"]),
13102 ..SeatHistory::default()
13103 };
13104 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13105 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13106 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13107 }
13108
13109 #[test]
13116 fn failure_signature_ignores_numbers_and_paths() {
13117 assert_eq!(
13118 failure_signature("exited with Some(2) and no usable output"),
13119 failure_signature("exited with Some(137) and no usable output")
13120 );
13121 assert_eq!(
13122 failure_signature("process exited with no usable output (exit code 2)"),
13123 failure_signature("process exited with no usable output (exit code 137)")
13124 );
13125 for (old, new) in [
13126 (
13127 "exited with Some(2) and no usable output",
13128 "process exited with no usable output (exit code 2)",
13129 ),
13130 (
13131 "exited with None and no usable output",
13132 "process exited with no usable output (exit code unavailable)",
13133 ),
13134 ] {
13135 let previous = FailClass::Other(failure_signature(old));
13136 let current = FailClass::Other(failure_signature(new));
13137 assert_eq!(previous, current, "resumption retains the failure class");
13138 assert!(!should_hand_over(Some(&previous), ¤t));
13139 }
13140 assert_eq!(
13141 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13142 failure_signature("cannot open /var/x.txt: denied")
13143 );
13144 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13145 }
13146
13147 #[test]
13148 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13149 let other = FailClass::Other("x".into());
13150 assert!(should_hand_over(None, &FailClass::Quota));
13151 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13152 assert!(should_hand_over(
13153 Some(&FailClass::Timeout),
13154 &FailClass::Timeout
13155 ));
13156 assert!(should_hand_over(None, &other));
13157 assert!(!should_hand_over(Some(&other), &other));
13158 assert!(should_hand_over(
13159 Some(&other),
13160 &FailClass::Other("y".into())
13161 ));
13162 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13164 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13165 }
13166
13167 #[test]
13168 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13169 let a = SeatState::new("judge-1", "alpha", 7);
13170 let b = handover_seat("judge-1", "beta", 7);
13171 assert_ne!(a.claude_session, b.claude_session);
13172 assert_eq!(b.turns, 0);
13173 }
13174
13175 #[test]
13176 fn a_timeout_is_classified_apart_from_other_failures() {
13177 assert_eq!(
13178 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13179 Some(FailClass::Timeout)
13180 );
13181 assert!(matches!(
13182 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13183 Some(FailClass::Other(_))
13184 ));
13185 }
13186
13187 #[test]
13188 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13189 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13190 let tried = BTreeSet::from(["beta".to_owned()]);
13191 let next = next_untried_in_roster(&roster, 1, &tried);
13194 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13195 }
13196
13197 #[test]
13198 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13199 let roster = vec![spec("alpha"), spec("beta")];
13200 let tried = BTreeSet::from(["beta".to_owned()]);
13201 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13205 }
13206
13207 #[test]
13208 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13209 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13210 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13211 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13215 }
13216
13217 #[test]
13218 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13219 let roster = vec![spec("a"), spec("a"), spec("b")];
13220 let tried = BTreeSet::from(["a".to_owned()]);
13221 let next = next_untried_in_roster(&roster, 0, &tried);
13222 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13223 }
13224
13225 #[test]
13226 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13227 let roster = vec![spec("a"), spec("b")];
13228 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13229 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13230 }
13231
13232 #[test]
13233 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13234 let dir = tempfile::tempdir().unwrap();
13235 let bay = dir.path().join("ffff");
13236
13237 remove_if_empty(&bay);
13239 assert!(!bay.exists());
13240
13241 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13244 remove_if_empty(&bay);
13245 assert!(bay.exists(), "non-empty directory must survive");
13246
13247 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13249 remove_if_empty(&bay);
13250 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13251 }
13252
13253 #[test]
13262 fn a_full_panel_that_found_nothing_is_clean() {
13263 assert!(round_is_clean(
13264 0,
13265 true,
13266 2,
13267 2,
13268 0,
13269 IncompleteReviewPolicy::Block
13270 ));
13271 }
13272
13273 #[test]
13274 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13275 assert!(!round_is_clean(
13276 0,
13277 true,
13278 1,
13279 2,
13280 0,
13281 IncompleteReviewPolicy::Block
13282 ));
13283 }
13284
13285 #[test]
13286 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13287 assert!(!round_is_clean(
13288 1,
13289 true,
13290 1,
13291 2,
13292 0,
13293 IncompleteReviewPolicy::Warn
13294 ));
13295 }
13296
13297 #[test]
13298 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13299 assert!(round_is_clean(
13300 0,
13301 true,
13302 1,
13303 2,
13304 0,
13305 IncompleteReviewPolicy::Warn
13306 ));
13307 }
13308
13309 #[test]
13310 fn a_full_panel_with_an_open_finding_is_not_clean() {
13311 assert!(!round_is_clean(
13312 1,
13313 true,
13314 2,
13315 2,
13316 0,
13317 IncompleteReviewPolicy::Block
13318 ));
13319 }
13320
13321 #[test]
13322 fn a_full_panel_with_a_red_e2e_is_not_clean() {
13323 assert!(!round_is_clean(
13324 0,
13325 false,
13326 2,
13327 2,
13328 0,
13329 IncompleteReviewPolicy::Block
13330 ));
13331 }
13332
13333 #[test]
13340 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13341 assert!(round_is_clean(
13344 0,
13345 true,
13346 1,
13347 2,
13348 1,
13349 IncompleteReviewPolicy::Block
13350 ));
13351 }
13352
13353 #[test]
13354 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13355 assert!(!round_is_clean(
13358 0,
13359 true,
13360 1,
13361 2,
13362 0,
13363 IncompleteReviewPolicy::Block
13364 ));
13365 }
13366
13367 #[test]
13368 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13369 assert!(!round_is_clean(
13370 1,
13371 true,
13372 1,
13373 2,
13374 1,
13375 IncompleteReviewPolicy::Block
13376 ));
13377 assert!(!round_is_clean(
13378 0,
13379 false,
13380 1,
13381 2,
13382 1,
13383 IncompleteReviewPolicy::Block
13384 ));
13385 }
13386
13387 #[test]
13388 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13389 assert!(!round_is_clean(
13393 0,
13394 true,
13395 0,
13396 2,
13397 2,
13398 IncompleteReviewPolicy::Block
13399 ));
13400 }
13401
13402 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13403 CommandOutcome {
13404 command: "test".to_owned(),
13405 code,
13406 output_tail: String::new(),
13407 duration_ms: 0,
13408 resource_blocked,
13409 }
13410 }
13411
13412 #[test]
13413 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13414 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13415 assert!(
13416 !verify_inconclusive(&[outcome(Some(1), false)]),
13417 "an ordinary failure is still evidence about the patch"
13418 );
13419 assert!(verify_inconclusive(&[outcome(None, true)]));
13420 assert!(
13421 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13422 "one inconclusive outcome taints the whole batch"
13423 );
13424 assert!(!verify_inconclusive(&[]));
13425 }
13426
13427 #[tokio::test]
13428 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13429 let calls = std::sync::atomic::AtomicUsize::new(0);
13433 let started = Instant::now();
13434 wait_for_pids_with(
13435 &[123],
13436 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13437 Duration::from_millis(5),
13438 Duration::from_secs(5),
13439 )
13440 .await;
13441 assert!(
13442 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13443 "must keep checking rather than deciding on the first answer"
13444 );
13445 assert!(
13446 started.elapsed() < Duration::from_secs(1),
13447 "must return the moment it is confirmed dead, not wait out the ceiling"
13448 );
13449 }
13450
13451 #[tokio::test]
13452 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13453 let started = Instant::now();
13454 wait_for_pids_with(
13455 &[123],
13456 |_| true, Duration::from_millis(5),
13458 Duration::from_millis(30),
13459 )
13460 .await;
13461 let elapsed = started.elapsed();
13462 assert!(
13463 elapsed >= Duration::from_millis(30),
13464 "must not give up before its own ceiling: {elapsed:?}"
13465 );
13466 assert!(
13467 elapsed < Duration::from_secs(1),
13468 "must not wait past its own ceiling either: {elapsed:?}"
13469 );
13470 }
13471
13472 #[tokio::test]
13473 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13474 let started = Instant::now();
13475 wait_for_pids_with(
13476 &[],
13477 |_| true,
13478 Duration::from_secs(5),
13479 Duration::from_secs(5),
13480 )
13481 .await;
13482 assert!(
13483 started.elapsed() < Duration::from_millis(200),
13484 "an empty pid list has nothing to confirm"
13485 );
13486 }
13487
13488 fn review_round(
13494 clean: bool,
13495 blocking: usize,
13496 answered: usize,
13497 expected: usize,
13498 progressed: bool,
13499 e2e_ok: bool,
13500 ) -> ReviewRound {
13501 ReviewRound {
13502 round: 1,
13503 head: "h".to_owned(),
13504 verified_head: None,
13505 verified_at: None,
13506 reviews: Vec::new(),
13507 e2e: vec![CommandOutcome {
13508 command: "test".to_owned(),
13509 code: Some(if e2e_ok { 0 } else { 1 }),
13510 output_tail: String::new(),
13511 duration_ms: 0,
13512 resource_blocked: false,
13513 }],
13514 verify_retried: false,
13515 e2e_deferred: false,
13516 e2e_defer_reason: None,
13517 fix: None,
13518 blocking,
13519 answered,
13520 expected,
13521 clean,
13522 progressed,
13523 vote_split: false,
13524 reconsideration: Vec::new(),
13525 verdict: None,
13526 }
13527 }
13528
13529 #[test]
13530 fn review_conclusion_is_none_when_nothing_has_run() {
13531 assert_eq!(review_conclusion(&[], 3), None);
13532 }
13533
13534 #[test]
13535 fn review_conclusion_is_none_while_rounds_remain() {
13536 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13537 assert_eq!(review_conclusion(&rounds, 3), None);
13538 }
13539
13540 #[test]
13541 fn review_conclusion_is_gating_once_a_round_is_clean() {
13542 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13543 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13544 }
13545
13546 #[test]
13547 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13548 let rounds = vec![
13549 review_round(false, 1, 2, 2, true, true),
13550 review_round(false, 1, 2, 2, true, true),
13551 ];
13552 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13553 }
13554
13555 #[test]
13556 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13557 let rounds = vec![
13558 review_round(false, 1, 2, 2, true, true),
13559 review_round(false, 1, 2, 2, true, false),
13560 ];
13561 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13562 }
13563
13564 #[test]
13565 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13566 let mut blocked = review_round(false, 1, 2, 2, true, false);
13573 blocked.e2e[0].resource_blocked = true;
13574 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13575 assert_eq!(review_conclusion(&rounds, 2), None);
13576 }
13577
13578 #[test]
13579 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13580 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13582 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13583 }
13584
13585 #[test]
13586 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13587 let rounds = vec![
13588 review_round(false, 1, 2, 2, false, true),
13589 review_round(false, 1, 2, 2, false, true),
13590 ];
13591 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13592 }
13593
13594 fn secs(n: u64) -> Duration {
13595 Duration::from_secs(n)
13596 }
13597
13598 #[tokio::test]
13601 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13602 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13603 if std::env::var_os(CHILD).is_none() {
13604 let output = std::process::Command::new(std::env::current_exe().unwrap())
13608 .args([
13609 "--exact",
13610 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13611 "--nocapture",
13612 ])
13613 .env(CHILD, "1")
13614 .env_remove("MAGI_WORKER_HOME")
13615 .env_remove("MAGI_WORKER_ID")
13616 .output()
13617 .unwrap();
13618 assert!(
13619 output.status.success()
13620 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13621 "{}\n{}",
13622 String::from_utf8_lossy(&output.stdout),
13623 String::from_utf8_lossy(&output.stderr)
13624 );
13625 return;
13626 }
13627 ask_test_home();
13628 let tmp = tempfile::tempdir().unwrap();
13629 let repo = tmp.path().join("repo");
13630 std::fs::create_dir_all(&repo).unwrap();
13631 init_repo(&repo);
13632 let g = |args: &[&str]| {
13633 let out = std::process::Command::new("git")
13634 .args(args)
13635 .current_dir(&repo)
13636 .quiet()
13637 .output()
13638 .unwrap();
13639 assert!(
13640 out.status.success(),
13641 "{}",
13642 String::from_utf8_lossy(&out.stderr)
13643 );
13644 };
13645 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13646 g(&["checkout", "-b", "historical"]);
13647 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13648 g(&["add", "."]);
13649 g(&["commit", "-m", "old alternative"]);
13650 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13651 g(&["checkout", "-b", "magi/task-selected", &base]);
13652 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13653 g(&["add", "."]);
13654 g(&["commit", "-m", "measurement infrastructure"]);
13655 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13656 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13657 g(&["add", "."]);
13658 g(&["commit", "-m", "dependent implementation"]);
13659 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13660 g(&["checkout", "main"]);
13661 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13662 g(&["add", "."]);
13663 g(&["commit", "-m", "unrelated base change"]);
13664 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13665 let origin = tmp.path().join("origin.git");
13666 g(&[
13667 "clone",
13668 "--bare",
13669 repo.to_str().unwrap(),
13670 origin.to_str().unwrap(),
13671 ]);
13672 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13673 g(&["fetch", "origin"]);
13674 let selection = crate::pr_recovery::Selection {
13675 url: "https://example.test/project/pull/1".into(),
13676 branch: "magi/task-selected".into(),
13677 head: head.clone(),
13678 source_head: None,
13679 source_run: None,
13680 source_proof: None,
13681 disposition: Some(crate::pr_recovery::Disposition::Review),
13682 instruction: "preserve measured behavior".into(),
13683 approved_at: jiff::Timestamp::now(),
13684 };
13685 let mut template = runner_at(RunStatus::Reviewing);
13686 template.state.repo = repo.clone();
13687 template.state.base_commit = current.clone();
13688 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13689 template.state.origin = Some(crate::run::Origin::queue("selected"));
13690 template.state.pr_head = Some(selection.branch.clone());
13691 template.state.pr_recovery = Some(selection.clone());
13692 let mut runner =
13693 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13694 .await
13695 .unwrap();
13696 let wt = runner.state.winner().unwrap().worktree.clone();
13697 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13698 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13699 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13700 assert!(runner.state.seeds.is_empty());
13701 assert_eq!(runner.state.instruction, "preserve measured behavior");
13702 runner.sync_to_base().await.unwrap();
13703 assert_eq!(
13704 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13705 "prerequisite"
13706 );
13707 assert_eq!(
13708 std::fs::read_to_string(wt.join("implementation")).unwrap(),
13709 "measured implementation"
13710 );
13711 assert_eq!(
13712 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13713 "current main change"
13714 );
13715 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13716 assert_eq!(
13717 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13718 head
13719 );
13720
13721 let mut partial = runner_at(RunStatus::Reviewing);
13723 partial.state.repo = repo.clone();
13724 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13725 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13726 run: Some("exit 1".into()),
13727 ..Default::default()
13728 }];
13729 let selection = runner.state.pr_recovery.clone().unwrap();
13730 partial.state.pr_recovery = Some(selection.clone());
13731 partial.state.pr_head = Some(selection.branch.clone());
13732 partial.state.origin = Some(crate::run::Origin::queue("selected"));
13733 let id = partial.state.id.clone();
13734 let error = match Runner::open_selected(
13735 &repo,
13736 partial.state,
13737 partial.roles,
13738 1,
13739 runner.state.base_commit.clone(),
13740 selection,
13741 )
13742 .await
13743 {
13744 Ok(_) => panic!("setup must fail"),
13745 Err(e) => e,
13746 };
13747 assert_eq!(
13748 error
13749 .downcast_ref::<crate::run::StartupFailure>()
13750 .unwrap()
13751 .run,
13752 id
13753 );
13754 let failed = RunState::load(&id).unwrap();
13755 assert_eq!(failed.status, RunStatus::Failed);
13756 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13757 assert!(Runner::resume(&id).is_err());
13758 runner.state.status = RunStatus::Ready;
13759 runner.state.driver_exited = true;
13760 runner.state.save().unwrap();
13761 let mut retry = runner_at(RunStatus::Reviewing);
13762 retry.state.repo = repo.clone();
13763 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13764 let mut reselected = runner.state.pr_recovery.clone().unwrap();
13765 reselected.approved_at = jiff::Timestamp::now();
13766 retry.state.origin = Some(crate::run::Origin::queue("selected"));
13767 retry.state.pr_head = Some(reselected.branch.clone());
13768 retry.state.pr_recovery = Some(reselected.clone());
13769 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13770 .await
13771 .unwrap();
13772 let restarted = Runner::open_selected(
13773 &repo,
13774 retry.state,
13775 retry.roles,
13776 1,
13777 runner.state.base_commit.clone(),
13778 reselected,
13779 )
13780 .await
13781 .unwrap();
13782 assert_eq!(
13783 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13784 .await
13785 .unwrap(),
13786 head
13787 );
13788 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13789 assert_eq!(
13790 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13791 head
13792 );
13793 }
13794
13795 #[tokio::test]
13796 #[cfg(unix)]
13797 async fn fallback_and_interrupted_restoration_require_coordination() {
13798 use std::os::fd::AsRawFd as _;
13799 ask_test_home();
13800 let tmp = tempfile::tempdir().unwrap();
13801 let mut runner = registered_runner(tmp.path()).await;
13802 runner.state.config.graph.timeout_resource_wait = 1;
13803 let winner = runner.state.candidates[0].clone();
13804 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13805 let leftover = winner.worktree.join("leftover");
13806 std::fs::write(&leftover, "untrusted edits").unwrap();
13807 let next = tmp.path().join("next-fixer");
13808 let mut failed = spec("first");
13809 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13810 let mut successor = spec("second");
13811 successor.command = vec![
13812 "sh".into(),
13813 "-c".into(),
13814 format!("touch '{}'; echo done", next.display()),
13815 ];
13816 let attempts = vec![
13817 (failed.clone(), "first".into()),
13818 (successor.clone(), "second".into()),
13819 ];
13820 let lock_path = runner.state.repo.join(".git/updater.lock");
13821 let external = std::fs::File::create(&lock_path).unwrap();
13822 assert_eq!(
13823 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13824 0
13825 );
13826 let artifacts = tmp.path().join("artifacts");
13827 let build = |spec, seat| SeatJob {
13828 spec,
13829 seat,
13830 cwd: winner.worktree.clone(),
13831 prompt: "fixture".into(),
13832 timeout: Duration::from_secs(10),
13833 allow_write: true,
13834 sessions: false,
13835 artifacts: artifacts.clone(),
13836 stem: "fix".into(),
13837 handover: None,
13838 };
13839 let (_, _, out) = runner
13840 .ask_fixer_among(
13841 attempts.clone(),
13842 "graft",
13843 None,
13844 Some((&winner.worktree, &head)),
13845 build,
13846 )
13847 .await;
13848 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13849 assert!(leftover.exists(), "contention must prevent clean");
13850 assert!(!next.exists());
13851 assert!(runner.state.handovers.is_empty());
13852 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13854 from_run: "earlier".into(),
13855 generation: 1,
13856 mode: crate::recovery::Mode::VerifyRepair,
13857 reasons: Vec::new(),
13858 reviewed_head: None,
13859 rounds: Vec::new(),
13860 open_findings: Vec::new(),
13861 evidence: None,
13862 criteria_inherited: true,
13863 repair: crate::recovery::Repair::Started {
13864 before: head.clone(),
13865 },
13866 });
13867 assert!(
13868 runner
13869 .recovery_repair(&winner)
13870 .await
13871 .unwrap_err()
13872 .is::<crate::repo_lock::ResourceBlocked>()
13873 );
13874 assert!(matches!(
13875 runner.state.recovery.as_ref().unwrap().repair,
13876 crate::recovery::Repair::Started { .. }
13877 ));
13878 assert!(leftover.exists());
13879 drop(external);
13880 runner.recovery_repair(&winner).await.unwrap();
13881 assert!(!leftover.exists());
13882 assert!(matches!(
13883 runner.state.recovery.as_ref().unwrap().repair,
13884 crate::recovery::Repair::Done { .. }
13885 ));
13886 std::fs::remove_file(&lock_path).unwrap();
13888 std::fs::create_dir(&lock_path).unwrap();
13889 std::fs::write(&leftover, "untrusted edits").unwrap();
13890 let (_, _, out) = runner
13891 .ask_fixer_among(
13892 attempts,
13893 "graft",
13894 None,
13895 Some((&winner.worktree, &head)),
13896 build,
13897 )
13898 .await;
13899 assert!(matches!(out, AgentOutcome::Failed(_)));
13900 assert!(leftover.exists());
13901 assert!(!next.exists());
13902 assert_eq!(
13903 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13904 head
13905 );
13906 }
13907
13908 #[cfg(unix)]
13909 async fn registered_runner(root: &Path) -> Runner {
13910 repository_runner(root, crate::config::test_coordination()).await
13911 }
13912
13913 async fn repository_runner(
13914 root: &Path,
13915 coordination: Option<crate::config::Coordination>,
13916 ) -> Runner {
13917 ask_test_home();
13918 let repo = root.join("repo");
13919 std::fs::create_dir_all(&repo).unwrap();
13920 init_repo(&repo);
13921 if coordination.is_some() {
13922 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13923 }
13924 let mut runner = runner_at(RunStatus::Prep);
13925 runner.state.repo = repo.clone();
13926 runner.state.config.coordination = coordination;
13927 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13928 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13929 runner.state.config.graph.advise = false;
13930 runner.roles.implementers = vec![spec("implement")];
13931 runner.roles.judges = vec![spec("judge")];
13932 runner.prep().await.unwrap();
13933 runner
13934 }
13935
13936 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
13937
13938 #[tokio::test]
13939 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
13940 let tmp = tempfile::tempdir().unwrap();
13941 let mut runner = repository_runner(tmp.path(), None).await;
13942 let prompts = runner.state.config.prompts.clone();
13943 let run = runner.state.id.clone();
13944 let ctx = WaveCtx {
13945 carry_seats: false,
13946 pauses: None,
13947 run: &run,
13948 node: "advise",
13949 prompts: &prompts,
13950 cache: None,
13951 round: None,
13952 };
13953 let jobs: Vec<_> = ["alpha", "beta"]
13954 .into_iter()
13955 .map(|id| SeatJob {
13956 spec: scripted(id, AUTH_OUTAGE),
13957 seat: SeatState::new(id, id, 7),
13958 prompt: "propose".into(),
13959 cwd: runner.state.candidates[0].worktree.clone(),
13960 timeout: Duration::from_secs(10),
13961 allow_write: false,
13962 sessions: false,
13963 artifacts: agent::artifacts_dir(&runner.state.dir()),
13964 stem: id.into(),
13965 handover: None,
13966 })
13967 .collect();
13968 let mut losses = Vec::new();
13969 let results = ask_wave_with(
13970 jobs.clone(),
13971 runner.sem.clone(),
13972 2,
13973 &[],
13974 &ctx,
13975 &mut losses,
13976 &mut runner.state,
13977 &|_: &str| Ok(()),
13978 )
13979 .await;
13980 assert!(
13981 results
13982 .iter()
13983 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
13984 );
13985 assert_eq!(runner.state.quota.len(), 2);
13986 assert!(
13987 runner
13988 .state
13989 .quota
13990 .iter()
13991 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
13992 );
13993 assert!(!runner.state.advise_attempted);
13994 runner.stop_for_resource().unwrap();
13995 assert_eq!(runner.state.status, RunStatus::Blocked);
13996 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
13997 assert_eq!(
13998 runner.state.resource_block.as_ref().unwrap().kind,
13999 "agent authentication"
14000 );
14001 runner.state.resource_block = None;
14004 runner.state.quota.clear();
14005 let mut mixed = jobs;
14006 mixed[1].spec.command = vec!["false".into()];
14007 let results = ask_wave_with(
14008 mixed,
14009 runner.sem.clone(),
14010 0,
14011 &[],
14012 &ctx,
14013 &mut losses,
14014 &mut runner.state,
14015 &|_: &str| Ok(()),
14016 )
14017 .await;
14018 assert!(results.iter().all(|(_, res, _)| res.is_err()));
14019 assert!(runner.state.resource_block.is_none());
14020 assert_eq!(losses.len(), 1);
14021 }
14022
14023 #[tokio::test]
14024 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14025 let tmp = tempfile::tempdir().unwrap();
14026 let mut runner = repository_runner(tmp.path(), None).await;
14027 let implementer = scripted("implement", AUTH_OUTAGE);
14028 runner.roles.implementers = vec![implementer.clone()];
14029 runner.state.config.agents = vec![implementer];
14030 runner.implement().await.unwrap();
14031 assert_eq!(runner.state.status, RunStatus::Blocked);
14032 assert_eq!(runner.state.quota.len(), 1);
14033 assert_eq!(
14034 runner.state.quota[0].cause,
14035 agent::QuotaCause::Authentication
14036 );
14037 let candidate = &runner.state.candidates[0];
14038 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14039 assert_eq!(runner.state.seats["impl-A"].turns, 1);
14040 }
14041
14042 #[tokio::test]
14043 #[cfg(unix)]
14044 async fn parked_implementer_persists_completed_work() {
14045 for registered in [false, true] {
14046 let tmp = tempfile::tempdir().unwrap();
14047 let mut runner = registered_runner(tmp.path()).await;
14048 if !registered {
14049 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14050 }
14051 let marker = tmp.path().join("started");
14052 let release = tmp.path().join("release");
14053 let mut implementer = spec("implement");
14054 implementer.command = vec![
14055 "sh".into(),
14056 "-c".into(),
14057 format!(
14058 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14059 marker.display(),
14060 release.display()
14061 ),
14062 ];
14063 runner.roles.implementers = vec![implementer.clone()];
14064 runner.state.config.agents = vec![implementer];
14065 let pause = runner.pause.clone();
14066 let (result, ()) = tokio::join!(runner.implement(), async {
14067 tokio::time::timeout(Duration::from_secs(10), async {
14068 while !marker.exists() {
14069 tokio::time::sleep(Duration::from_millis(10)).await;
14070 }
14071 })
14072 .await
14073 .unwrap();
14074 pause.park();
14075 std::fs::write(&release, "go").unwrap();
14076 });
14077 result.unwrap();
14078 let candidate = &runner.state.candidates[0];
14079 assert!(candidate.commits > 0, "{candidate:?}");
14080 assert!(candidate.summary.contains("saved-result"));
14081 assert_eq!(candidate.files, 1);
14082 let before = candidate.commits;
14083 runner.implement().await.unwrap();
14084 assert_eq!(runner.state.candidates[0].commits, before);
14085 }
14086 }
14087
14088 #[tokio::test]
14089 #[cfg(unix)]
14090 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14091 let tmp = tempfile::tempdir().unwrap();
14092 let mut runner = registered_runner(tmp.path()).await;
14093 let winner = runner.state.candidates[0].clone();
14094 let cache = tmp.path().join("cache");
14095 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14096 runner.state.config.graph.timeout_resource_wait = 0;
14097 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14098 let crate::cache::AcquireOutcome::Acquired(held) =
14099 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14100 else {
14101 panic!("cache must be free")
14102 };
14103 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14104 from_run: "earlier".into(),
14105 generation: 1,
14106 mode: crate::recovery::Mode::VerifyRepair,
14107 reasons: Vec::new(),
14108 reviewed_head: None,
14109 rounds: Vec::new(),
14110 open_findings: Vec::new(),
14111 evidence: None,
14112 criteria_inherited: true,
14113 repair: crate::recovery::Repair::NotStarted,
14114 });
14115 runner.recovery_repair(&winner).await.unwrap();
14116 assert!(matches!(
14117 runner.state.recovery.as_ref().unwrap().repair,
14118 crate::recovery::Repair::WaitingForRestore { .. }
14119 ));
14120 drop(held);
14121 runner.state.resource_block = None;
14122 runner.state.config.graph.timeout_resource_wait = 1;
14123 runner.recovery_repair(&winner).await.unwrap();
14124 assert!(matches!(
14125 runner.state.recovery.as_ref().unwrap().repair,
14126 crate::recovery::Repair::Done { .. }
14127 ));
14128 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14129 }
14130
14131 #[tokio::test]
14132 #[cfg(unix)]
14133 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14134 use std::os::fd::AsRawFd as _;
14135 for graft in [false, true] {
14136 let tmp = tempfile::tempdir().unwrap();
14137 let mut runner = registered_runner(tmp.path()).await;
14138 runner.state.config.graph.timeout_resource_wait = 1;
14139 let winner = runner.state.candidates[0].clone();
14140 let marker = tmp.path().join("fixer-started");
14141 let mut fixer = spec("fixer");
14142 fixer.command = vec![
14143 "sh".into(),
14144 "-c".into(),
14145 format!("echo started >> '{}'; echo done", marker.display()),
14146 ];
14147 runner.roles.fixer = Some(vec![fixer]);
14148 if graft {
14149 let mut loser = winner.clone();
14150 loser.label = 'B';
14151 loser.index = 1;
14152 runner.state.candidates.push(loser);
14153 runner.state.config.graph.graft_runner_up = true;
14154 runner.state.tally = Some(Tally {
14155 first_choice: BTreeMap::new(),
14156 borda: BTreeMap::new(),
14157 winner: winner.label,
14158 rankings: 1,
14159 unanimous_initial: true,
14160 deliberated: false,
14161 changed_votes: 0,
14162 unanimous_final: true,
14163 tie_break: None,
14164 judges: 1,
14165 present: 1,
14166 quorum: 1,
14167 met_quorum: true,
14168 uncontested: None,
14169 });
14170 runner.state.judgements.push(crate::run::Judgement {
14171 judge: 1,
14172 seat: "judge-1".into(),
14173 agent: "judge".into(),
14174 ranking: vec![winner.label, 'B'],
14175 reasons: BTreeMap::new(),
14176 confidence: None,
14177 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14178 order: vec![0, 1],
14179 failed: None,
14180 duration_ms: 0,
14181 });
14182 } else {
14183 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14184 from_run: "earlier".into(),
14185 generation: 1,
14186 mode: crate::recovery::Mode::VerifyRepair,
14187 reasons: Vec::new(),
14188 reviewed_head: None,
14189 rounds: Vec::new(),
14190 open_findings: Vec::new(),
14191 evidence: None,
14192 criteria_inherited: true,
14193 repair: crate::recovery::Repair::NotStarted,
14194 });
14195 }
14196 let reader = crate::repo_lock::SeatLease::for_checkout(
14197 &winner.worktree,
14198 false,
14199 crate::config::test_coordination().as_ref(),
14200 )
14201 .await
14202 .unwrap();
14203 let pause = runner.pause.clone();
14204 let (result, ()) = tokio::join!(
14205 async {
14206 if graft {
14207 runner.graft().await
14208 } else {
14209 runner.recovery_repair(&winner).await
14210 }
14211 },
14212 async {
14213 tokio::time::sleep(Duration::from_millis(100)).await;
14214 pause.park();
14215 }
14216 );
14217 result.unwrap();
14218 assert!(!marker.exists(), "a blocked seat must not spawn");
14219 runner.state = RunState::load(&runner.state.id).unwrap();
14221 if graft {
14222 assert_eq!(
14223 runner.state.graft.as_ref().unwrap().outcome,
14224 GraftOutcome::WaitingForRestore
14225 );
14226 } else {
14227 assert!(matches!(
14228 runner.state.recovery.as_ref().unwrap().repair,
14229 crate::recovery::Repair::WaitingForRestore { .. }
14230 ));
14231 }
14232 drop(reader);
14233 runner.pause = Pause::new();
14234 runner.state.resource_block = None;
14235 let external =
14237 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14238 assert_eq!(
14239 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14240 0
14241 );
14242 let leftover = winner.worktree.join("leftover");
14243 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14244 let result = if graft {
14245 runner.graft().await
14246 } else {
14247 runner.recovery_repair(&winner).await
14248 };
14249 assert!(
14250 result
14251 .unwrap_err()
14252 .is::<crate::repo_lock::ResourceBlocked>()
14253 );
14254 assert!(leftover.exists());
14255 assert!(!marker.exists());
14256 drop(external);
14257 runner.state.resource_block = None;
14258 let result = if graft {
14259 runner.graft().await
14260 } else {
14261 runner.recovery_repair(&winner).await
14262 };
14263 result.unwrap();
14264 assert!(!leftover.exists(), "restore must precede retry");
14265 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14266 if graft {
14267 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14268 } else {
14269 assert!(matches!(
14270 runner.state.recovery.as_ref().unwrap().repair,
14271 crate::recovery::Repair::Done { .. }
14272 ));
14273 }
14274 }
14275 }
14276
14277 #[tokio::test]
14278 #[cfg(unix)]
14279 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14280 use std::os::fd::AsRawFd as _;
14281 let tmp = tempfile::tempdir().unwrap();
14282 let mut runner = registered_runner(tmp.path()).await;
14283 runner.state.config.graph.timeout_resource_wait = 1;
14284 let winner = runner.state.candidates[0].clone();
14285 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14286 runner.state.tally = Some(Tally {
14287 first_choice: BTreeMap::new(),
14288 borda: BTreeMap::new(),
14289 winner: winner.label,
14290 rankings: 0,
14291 unanimous_initial: false,
14292 deliberated: false,
14293 changed_votes: 0,
14294 unanimous_final: false,
14295 tie_break: None,
14296 judges: 0,
14297 present: 0,
14298 quorum: 0,
14299 met_quorum: true,
14300 uncontested: Some("fixture".into()),
14301 });
14302 runner.state.graft = Some(GraftRecord {
14303 pre: pre.clone(),
14304 post: None,
14305 lines: None,
14306 limit: None,
14307 agent: None,
14308 outcome: GraftOutcome::Running,
14309 requested: Vec::new(),
14310 adoption: None,
14311 adoption_error: None,
14312 });
14313 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14314 git::commit_all(&winner.worktree, "unfinished graft")
14315 .await
14316 .unwrap();
14317 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14318 let leftover = winner.worktree.join("leftover");
14319 std::fs::write(&leftover, "untrusted edits").unwrap();
14320 let path = runner.state.repo.join(".git/updater.lock");
14321 let external = std::fs::File::create(&path).unwrap();
14322 assert_eq!(
14323 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14324 0
14325 );
14326 assert!(
14327 runner
14328 .graft()
14329 .await
14330 .unwrap_err()
14331 .is::<crate::repo_lock::ResourceBlocked>()
14332 );
14333 assert!(matches!(
14334 runner.state.graft.as_ref().unwrap().outcome,
14335 GraftOutcome::Running
14336 ));
14337 assert_eq!(
14338 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14339 unfinished
14340 );
14341 assert!(leftover.exists());
14342 drop(external);
14343 std::fs::remove_file(&path).unwrap();
14344 std::fs::create_dir(&path).unwrap();
14345 assert!(runner.graft().await.is_err());
14346 assert!(matches!(
14347 runner.state.graft.as_ref().unwrap().outcome,
14348 GraftOutcome::Running
14349 ));
14350 assert_eq!(
14351 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14352 unfinished
14353 );
14354 assert!(leftover.exists());
14355 std::fs::remove_dir(&path).unwrap();
14356 runner.graft().await.unwrap();
14357 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14358 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
14359 assert!(!leftover.exists());
14360 }
14361
14362 #[tokio::test]
14363 #[cfg(unix)]
14364 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
14365 use std::os::fd::AsRawFd as _;
14366 let tmp = tempfile::tempdir().unwrap();
14367 let mut runner = registered_runner(tmp.path()).await;
14368 runner.state.prep_complete = Some(false);
14369 runner.state.config.graph.timeout_resource_wait = 1;
14370 runner.state.save().unwrap();
14371 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14372 assert_eq!(
14373 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14374 0
14375 );
14376 runner.execute().await.unwrap();
14377 let disk = RunState::load(&runner.state.id).unwrap();
14378 assert!(disk.driver_exited);
14379 assert_eq!(disk.driver_pid, None);
14380 assert_eq!(runner.state.status, RunStatus::Blocked);
14381 assert_eq!(
14382 runner.state.resource_block.as_ref().unwrap().kind,
14383 "repository coordination"
14384 );
14385 assert!(runner.state.prep_failure.is_none());
14386 assert!(runner.state.seats.is_empty());
14387 runner.pause.park();
14388 tokio::time::timeout(Duration::from_secs(10), runner.execute())
14391 .await
14392 .unwrap()
14393 .unwrap();
14394 assert!(runner.state.parked);
14395 assert!(runner.state.prep_failure.is_none());
14396 let disk = RunState::load(&runner.state.id).unwrap();
14397 assert!(disk.driver_exited);
14398 assert_eq!(disk.driver_pid, None);
14399 }
14400
14401 #[tokio::test]
14402 #[cfg(unix)]
14403 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14404 use std::os::fd::AsRawFd as _;
14405 let tmp = tempfile::tempdir().unwrap();
14406 let mut runner = registered_runner(tmp.path()).await;
14407 runner.state.prep_complete = Some(false);
14408 runner.state.config.graph.timeout_resource_wait = 30;
14409 runner.state.save().unwrap();
14410 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14411 assert_eq!(
14412 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14413 0
14414 );
14415 assert!(
14416 tokio::time::timeout(Duration::from_millis(100), runner.execute())
14417 .await
14418 .is_err()
14419 );
14420 let disk = RunState::load(&runner.state.id).unwrap();
14421 assert!(disk.driver_exited);
14422 assert_eq!(disk.driver_pid, None);
14423 assert!(disk.status.resumable());
14424 assert!(runner.state.driver_exited);
14425 assert_eq!(runner.state.driver_pid, None);
14426 runner.state.save().unwrap();
14427 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14428 }
14429
14430 #[tokio::test]
14431 #[cfg(unix)]
14432 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14433 let tmp = tempfile::tempdir().unwrap();
14434 let mut runner = registered_runner(tmp.path()).await;
14435 let winner = runner.state.candidates[0].clone();
14436 let cache = tmp.path().join("cache");
14437 let started = tmp.path().join("pre-gate-started");
14438 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14439 runner.state.config.verify.gate =
14440 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14441 let home = crate::run::home();
14442 let owner =
14443 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14444 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14445 crate::cache::AcquireOutcome::Acquired(g) => g,
14446 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14447 };
14448 let mut gate = Box::pin(runner.run_gate(&winner));
14449 assert!(
14450 tokio::time::timeout(Duration::from_millis(150), &mut gate)
14451 .await
14452 .is_err()
14453 );
14454 assert!(!started.exists(), "pre-gate must wait for the cache too");
14455 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14459 let editor = crate::repo_lock::SeatLease::checkout_with(
14460 &winner.worktree,
14461 true,
14462 crate::config::test_coordination().as_ref(),
14463 &wait,
14464 )
14465 .await
14466 .unwrap();
14467 drop(editor);
14468 guard.release();
14469 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14470 .await
14471 .unwrap()
14472 .unwrap();
14473 drop(gate);
14474 assert!(outcomes.iter().all(CommandOutcome::ok));
14475 assert!(started.exists());
14476 }
14477
14478 #[tokio::test]
14479 #[cfg(unix)]
14480 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14481 use std::os::fd::AsRawFd as _;
14482
14483 ask_test_home();
14484 let tmp = tempfile::tempdir().unwrap();
14485 let repo = tmp.path().join("repo");
14486 std::fs::create_dir_all(&repo).unwrap();
14487 init_repo(&repo);
14488 let common = repo.join(".git");
14489 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14490 let mut runner = runner_at(RunStatus::Prep);
14491 runner.state.repo = repo.clone();
14492 runner.state.config.coordination = crate::config::test_coordination();
14493 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14494 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14495 runner.state.config.graph.advise = false;
14496 runner.roles.implementers = vec![spec("implement")];
14497 runner.roles.judges = vec![spec("judge")];
14498 runner.prep().await.unwrap();
14499 let winner = runner.state.candidates[0].clone();
14500 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14503 let completed = tmp.path().join("seat-completed");
14504 runner.roles.implementers[0].command = vec![
14505 "sh".into(),
14506 "-c".into(),
14507 format!("touch '{}'; echo completed", completed.display()),
14508 ];
14509 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14510 assert_eq!(
14511 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14512 0
14513 );
14514 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14515 let mut implement = Box::pin(runner.implement());
14516 tokio::select! {
14517 result = &mut implement => panic!("rescue must wait: {result:?}"),
14518 result = tokio::time::timeout(Duration::from_secs(10), async {
14519 while !completed.exists() {
14520 tokio::time::sleep(Duration::from_millis(20)).await;
14521 }
14522 }) => result.unwrap(),
14523 }
14524 assert!(
14525 tokio::time::timeout(Duration::from_millis(200), &mut implement)
14526 .await
14527 .is_err()
14528 );
14529 assert_eq!(
14530 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14531 before
14532 );
14533 assert_eq!(
14534 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14535 0
14536 );
14537 tokio::time::timeout(Duration::from_secs(10), &mut implement)
14538 .await
14539 .unwrap()
14540 .unwrap();
14541 drop(implement);
14542 assert_ne!(
14543 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14544 before
14545 );
14546 assert!(git::is_clean(&winner.worktree).await.unwrap());
14547
14548 let started = tmp.path().join("pre-gate-started");
14549 let proceed = tmp.path().join("pre-gate-proceed");
14550 let checked = tmp.path().join("gate-ran");
14551 runner.state.config.verify.pre_gate = vec![format!(
14552 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14553 started.display(),
14554 proceed.display()
14555 )];
14556 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14557 assert_eq!(
14558 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14559 0
14560 );
14561 let mut gate = Box::pin(runner.run_gate(&winner));
14562 assert!(
14563 tokio::time::timeout(Duration::from_millis(200), &mut gate)
14564 .await
14565 .is_err()
14566 );
14567 assert!(!started.exists());
14568 assert!(!checked.exists());
14569 assert!(git::is_clean(&winner.worktree).await.unwrap());
14570 assert_eq!(
14571 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14572 0
14573 );
14574 tokio::select! {
14575 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14576 result = tokio::time::timeout(Duration::from_secs(10), async {
14577 while !started.exists() {
14578 tokio::time::sleep(Duration::from_millis(20)).await;
14579 }
14580 }) => result.unwrap(),
14581 }
14582 assert_eq!(
14583 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14584 -1,
14585 "mechanical edits must hold the real lock"
14586 );
14587 std::fs::write(proceed, "ready").unwrap();
14588 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14589 .await
14590 .unwrap()
14591 .unwrap();
14592 drop(gate);
14593 assert!(outcomes.iter().all(CommandOutcome::ok));
14594 assert!(checked.exists());
14595 assert!(runner.state.pre_gate_commit.is_some());
14596 assert!(git::is_clean(&winner.worktree).await.unwrap());
14597 assert_eq!(
14598 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14599 0,
14600 "gate completion must release coordination"
14601 );
14602 drop(outsider);
14603 std::fs::remove_file(common.join("updater.lock")).unwrap();
14604 std::fs::create_dir(common.join("updater.lock")).unwrap();
14605 std::fs::remove_file(&started).unwrap();
14606 std::fs::remove_file(&checked).unwrap();
14607 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14608 let error = runner.run_gate(&winner).await.unwrap_err();
14609 assert!(
14610 error
14611 .to_string()
14612 .contains("acquire mandatory repository lock")
14613 );
14614 assert!(
14615 !started.exists(),
14616 "failed acquisition must prevent pre-gate"
14617 );
14618 assert!(!checked.exists(), "failed acquisition must prevent gate");
14619 assert!(git::is_clean(&winner.worktree).await.unwrap());
14620 assert_eq!(
14621 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14622 head
14623 );
14624 }
14625
14626 #[tokio::test]
14627 #[cfg(unix)]
14628 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14629 use std::os::fd::AsRawFd as _;
14630 use std::time::Duration;
14631
14632 ask_test_home();
14633 let tmp = tempfile::tempdir().unwrap();
14634 let repo = tmp.path().join("repo");
14635 std::fs::create_dir_all(&repo).unwrap();
14636 init_repo(&repo);
14637 let common = repo.join(".git");
14638 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14639 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14640 assert_eq!(
14641 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14642 0
14643 );
14644 let mut runner = runner_at(RunStatus::Prep);
14645 runner.state.repo = repo.clone();
14646 runner.state.config.coordination = crate::config::test_coordination();
14647 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14648 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14649 runner.state.config.graph.advise = true;
14650 runner.state.config.graph.advisors = 1;
14651 runner.roles.implementers = vec![spec("implement")];
14652 runner.roles.judges = vec![spec("judge")];
14653 let root = runner.state.worktree_root();
14654 let started = tmp.path().join("setup-started");
14655 let proceed = tmp.path().join("setup-proceed");
14656 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14657 run: Some(format!(
14658 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14659 started.display(),
14660 proceed.display()
14661 )),
14662 ..Default::default()
14663 }];
14664 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14665 let config = git::git(&repo, &["config", "--local", "--list"])
14666 .await
14667 .unwrap();
14668 let mut prep = Box::pin(runner.prep());
14669 assert!(
14670 tokio::time::timeout(Duration::from_millis(200), &mut prep)
14671 .await
14672 .is_err()
14673 );
14674 assert!(
14675 !root.exists(),
14676 "contention must prevent even checkout creation"
14677 );
14678 assert!(!started.exists(), "contention must prevent setup");
14679 assert_eq!(
14680 git::git(&repo, &["branch", "--list"]).await.unwrap(),
14681 branches
14682 );
14683 assert_eq!(
14684 git::git(&repo, &["config", "--local", "--list"])
14685 .await
14686 .unwrap(),
14687 config
14688 );
14689 drop(lock);
14690 tokio::select! {
14691 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14692 result = tokio::time::timeout(Duration::from_secs(10), async {
14693 while !started.exists() {
14694 tokio::time::sleep(Duration::from_millis(20)).await;
14695 }
14696 }) => result.unwrap(),
14697 }
14698 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14699 assert_eq!(
14700 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14701 -1
14702 );
14703 assert!(
14704 tokio::time::timeout(
14705 Duration::from_millis(100),
14706 crate::repo_lock::SeatLease::for_checkout(
14707 &repo,
14708 true,
14709 crate::config::test_coordination().as_ref()
14710 )
14711 )
14712 .await
14713 .is_err(),
14714 "preparation must also exclude local editors"
14715 );
14716 std::fs::write(proceed, "ready").unwrap();
14717 tokio::time::timeout(Duration::from_secs(10), &mut prep)
14718 .await
14719 .unwrap()
14720 .unwrap();
14721 drop(prep);
14722 assert_eq!(runner.state.prep_complete, Some(true));
14723 assert!(root.join("judge-1").exists());
14724 assert!(root.join("advisor-1").exists());
14725 assert_eq!(
14726 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14727 0
14728 );
14729 }
14730
14731 #[tokio::test]
14732 #[cfg(unix)]
14733 async fn preparation_lock_failure_prevents_repository_changes() {
14734 ask_test_home();
14735 let tmp = tempfile::tempdir().unwrap();
14736 let repo = tmp.path().join("repo");
14737 std::fs::create_dir_all(&repo).unwrap();
14738 init_repo(&repo);
14739 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14740 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14743 let mut runner = runner_at(RunStatus::Prep);
14744 runner.state.repo = repo.clone();
14745 runner.state.config.coordination = crate::config::test_coordination();
14746 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14747 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14748 runner.roles.implementers = vec![spec("implement")];
14749 let error = runner.prep().await.unwrap_err();
14750 assert!(
14751 error
14752 .to_string()
14753 .contains("acquire mandatory repository lock")
14754 );
14755 assert!(!runner.state.worktree_root().exists());
14756 assert!(runner.state.candidates.is_empty());
14757 assert!(!repo.join(".git/config.worktree").exists());
14758 assert_eq!(
14759 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14760 "* main"
14761 );
14762 }
14763
14764 #[tokio::test]
14765 async fn preparation_records_creation_intent_before_git_can_fail() {
14766 ask_test_home();
14767 let tmp = tempfile::tempdir().unwrap();
14768 let repo = tmp.path().join("repo");
14769 std::fs::create_dir_all(&repo).unwrap();
14770 init_repo(&repo);
14771 let mut runner = runner_at(RunStatus::Prep);
14772 runner.state.repo = repo.clone();
14773 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14774 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14775 runner.state.config.blind.commit_msg_hook = false;
14776 runner.roles.implementers = vec![spec("one")];
14777 let label = blind::assign_labels(1, runner.state.seed)[0];
14778 let branch = runner.state.branch_for(label);
14779 git::git(&repo, &["branch", &branch]).await.unwrap();
14782 assert!(runner.prep().await.is_err());
14783 let saved = RunState::load(&runner.state.id).unwrap();
14784 assert!(saved.candidates.is_empty());
14785 let event = saved
14786 .events
14787 .iter()
14788 .find(|e| e.node == "prep-intent")
14789 .unwrap();
14790 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14791 assert_eq!(intent["branch"], branch);
14792 assert_eq!(intent["start"], runner.state.base_commit);
14793 assert_eq!(
14794 intent["worktree"],
14795 runner
14796 .state
14797 .worktree_root()
14798 .join(format!("cand-{label}"))
14799 .to_string_lossy()
14800 .as_ref()
14801 );
14802 assert_eq!(intent["picks"], serde_json::json!([]));
14803 }
14804
14805 #[tokio::test]
14806 async fn invalid_pinned_sources_create_no_partial_worktrees() {
14807 ask_test_home();
14808 let tmp = tempfile::tempdir().unwrap();
14809 let repo = tmp.path().join("repo");
14810 std::fs::create_dir_all(&repo).unwrap();
14811 init_repo(&repo);
14812 let mut runner = runner_at(RunStatus::Prep);
14813 runner.state.repo = repo.clone();
14814 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14815 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14816 runner.state.sources.pick_commits.push(refs::Input {
14817 token: "missing".into(),
14818 sha: "0".repeat(40),
14819 provenance: refs::Provenance::ExplicitCli,
14820 });
14821 assert!(runner.prep().await.is_err());
14822 assert!(runner.state.candidates.is_empty());
14823 assert!(!runner.state.worktree_root().exists());
14824 }
14825
14826 #[tokio::test]
14827 async fn prose_and_legacy_inference_never_import_into_candidates() {
14828 ask_test_home();
14829 let tmp = tempfile::tempdir().unwrap();
14830 let repo = tmp.path().join("repo");
14831 std::fs::create_dir_all(&repo).unwrap();
14832 init_repo(&repo);
14833 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14834 git::git(&repo, &["checkout", "-b", "magi/history/A"])
14835 .await
14836 .unwrap();
14837 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14838 git::git(&repo, &["add", "."]).await.unwrap();
14839 git::git(&repo, &["commit", "-m", "historical implementation"])
14840 .await
14841 .unwrap();
14842 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14843 git::git(&repo, &["checkout", "main"]).await.unwrap();
14844 for instruction in [
14845 format!("Example: magi/history/A at {source}"),
14846 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14847 format!("Never import magi/history/A or {source}"),
14848 format!("Conversation answer: I mentioned {source} only as background"),
14849 ] {
14850 let mut runner = runner_at(RunStatus::Prep);
14851 runner.state.repo = repo.clone();
14852 runner.state.base_commit = base.clone();
14853 runner.state.instruction = instruction.clone();
14854 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14855 runner.state.config.blind.commit_msg_hook = false;
14856 runner.state.seeds = vec![refs::Seed {
14857 token: "magi/history/A".into(),
14858 sha: source.clone(),
14859 kind: refs::SeedKind::Unmerged,
14860 branch: true,
14861 detail: String::new(),
14862 }];
14863 runner.roles.implementers = vec![spec("one")];
14864 assert_eq!(seeded_instruction(&runner.state), instruction);
14865 let mut legacy = serde_json::to_value(&runner.state).unwrap();
14867 legacy.as_object_mut().unwrap().remove("sources");
14868 runner.state = serde_json::from_value(legacy).unwrap();
14869 assert_eq!(runner.state.sources, refs::Sources::default());
14870 runner.state.save().unwrap();
14871 runner.state = RunState::load(&runner.state.id).unwrap();
14872 runner.prep().await.unwrap();
14873 for candidate in &runner.state.candidates {
14874 assert_eq!(
14875 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14876 base
14877 );
14878 assert!(!candidate.worktree.join("unwanted").exists());
14879 }
14880 }
14881 }
14882
14883 #[tokio::test]
14884 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14885 ask_test_home();
14886 let tmp = tempfile::tempdir().unwrap();
14887 let repo = tmp.path().join("repo");
14888 std::fs::create_dir_all(&repo).unwrap();
14889 init_repo(&repo);
14890 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14891 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14892 std::fs::write(repo.join("referenced"), "required commit").unwrap();
14893 git::git(&repo, &["add", "."]).await.unwrap();
14894 git::git(&repo, &["commit", "-m", "required seed"])
14895 .await
14896 .unwrap();
14897 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14898 git::git(&repo, &["checkout", "main"]).await.unwrap();
14899 let mut runner = runner_at(RunStatus::Prep);
14900 runner.state.repo = repo.clone();
14901 runner.state.base_commit = base.clone();
14902 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14903 runner.state.config.graph.advise = false;
14904 runner.state.config.blind.commit_msg_hook = false;
14905 runner.state.sources =
14906 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14907 .await
14908 .unwrap();
14909 runner.roles.implementers = vec![spec("one"), spec("two")];
14910 runner.roles.judges = vec![spec("judge")];
14911 let label = blind::assign_labels(2, runner.state.seed)[0];
14912 let branch = runner.state.branch_for(label);
14913 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14914 git::worktree_add_branch(&repo, &worktree, &branch, &base)
14915 .await
14916 .unwrap();
14917 runner.state.candidates.push(Candidate {
14918 index: 0,
14919 label,
14920 agent: "one".into(),
14921 branch,
14922 worktree,
14923 summary: String::new(),
14924 stat: String::new(),
14925 files: 0,
14926 commits: 0,
14927 empty: false,
14928 failed: None,
14929 verified_noop: None,
14930 duration_ms: 0,
14931 folded: false,
14932 });
14933 runner.state.save().unwrap();
14934 runner.state = RunState::load(&runner.state.id).unwrap();
14936 assert_eq!(runner.state.prep_complete, Some(false));
14937 runner.prep().await.unwrap();
14938 assert_eq!(runner.state.prep_complete, Some(true));
14939 assert_eq!(runner.state.candidates.len(), 2);
14940 for candidate in &runner.state.candidates {
14941 assert_eq!(
14942 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14943 git::tree_of(&repo, &seed).await.unwrap()
14944 );
14945 assert_eq!(
14946 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14947 "required commit"
14948 );
14949 }
14950 assert!(runner.state.worktree_root().join("judge-1").exists());
14951 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14952 runner.prep().await.unwrap();
14953 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14954
14955 runner.state.prep_complete = Some(false);
14956 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14957 run: Some(format!(
14958 "test -f '{}'",
14959 tmp.path().join("dependency-ready").display()
14960 )),
14961 ..Default::default()
14962 }];
14963 assert!(runner.prep().await.is_err());
14964 assert!(runner.state.prep_failure.is_some());
14965 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14966 runner.state = RunState::load(&runner.state.id).unwrap();
14967 assert!(
14968 runner.prep().await.is_err(),
14969 "correcting the cause alone must not auto-retry"
14970 );
14971 let failed_at = runner.state.events.last().unwrap().at;
14972 assert!(
14973 runner.retry_preparation(failed_at).is_err(),
14974 "old approvals cannot unlock the hold"
14975 );
14976 let mut task = crate::queue::Task::new(
14977 "retry".into(),
14978 "retry setup".into(),
14979 repo.clone(),
14980 crate::queue::Source::Human,
14981 );
14982 task.start(runner.state.id.clone());
14983 task.hold_machine(Some("setup failed".into()));
14984 task.release_operator("cli", None);
14985 runner
14986 .retry_preparation(task.resumption.as_ref().unwrap().at)
14987 .unwrap();
14988 runner.prep().await.unwrap();
14989 assert!(runner.state.prep_failure.is_none());
14990 assert_eq!(runner.state.prep_complete, Some(true));
14991 assert_eq!(runner.state.candidates.len(), 2);
14992 assert!(runner.state.worktree_root().join("judge-1").exists());
14993 for candidate in &runner.state.candidates {
14994 assert_eq!(
14995 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14996 git::tree_of(&repo, &seed).await.unwrap()
14997 );
14998 }
14999 }
15000
15001 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15002 let mut heads = Vec::new();
15003 for candidate in candidates {
15004 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15005 }
15006 heads
15007 }
15008
15009 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15010 ask_test_home();
15011 let tmp = tempfile::tempdir().unwrap();
15012 let repo = tmp.path().join("repo");
15013 std::fs::create_dir(&repo).unwrap();
15014 init_repo(&repo);
15015 let mut runner = runner_at(RunStatus::Reviewing);
15016 runner.state.repo = repo.clone();
15017 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15018 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15019 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15020 let branch = format!("candidate-{label}");
15021 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15022 git::worktree_add_branch(&repo, &worktree, &branch, "main")
15023 .await
15024 .unwrap();
15025 if label == 'B' {
15026 std::fs::write(worktree.join("change"), "winner").unwrap();
15027 git::rescue_commit(&worktree, "candidate change")
15028 .await
15029 .unwrap();
15030 }
15031 runner.state.candidates.push(Candidate {
15032 index,
15033 label,
15034 agent: "alpha".into(),
15035 branch,
15036 worktree,
15037 summary: "completed reply".into(),
15038 stat: String::new(),
15039 files: usize::from(label == 'B'),
15040 commits: usize::from(label == 'B'),
15041 empty: label == 'A',
15042 failed: None,
15043 verified_noop: None,
15044 duration_ms: 1,
15045 folded: false,
15046 });
15047 }
15048 runner.state.judge_skipped = true;
15049 runner.state.tally = Some(Tally {
15050 first_choice: BTreeMap::new(),
15051 borda: BTreeMap::new(),
15052 winner: 'B',
15053 rankings: 0,
15054 unanimous_initial: false,
15055 deliberated: false,
15056 changed_votes: 0,
15057 unanimous_final: false,
15058 tie_break: None,
15059 judges: 0,
15060 present: 0,
15061 quorum: 0,
15062 met_quorum: true,
15063 uncontested: Some("only B produced a change".into()),
15064 });
15065 runner.state.save().unwrap();
15066 (tmp, runner)
15067 }
15068
15069 #[tokio::test]
15070 #[cfg(unix)]
15071 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15072 for (solo, failed_checkpoint) in
15073 [(true, false), (false, false), (true, true), (false, true)]
15074 {
15075 let (_tmp, mut runner) = preservation_resume_fixture().await;
15076 if solo {
15077 runner.state.candidates.retain(|c| c.label == 'A');
15078 }
15079 runner.state.candidates[0].empty = false;
15080 runner.state.candidates[0].summary.clear();
15081 runner.state.tally = None;
15082 runner.state.judge_skipped = false;
15083 runner.state.status = RunStatus::Implementing;
15084 runner.state.config.disk.min_free_bytes = u64::MAX;
15085 let mut implementer = spec("alpha");
15086 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15087 runner.roles.implementers = vec![implementer];
15088 let patch = runner.state.dir().join("artifacts/cand-A.patch");
15089 if failed_checkpoint {
15090 std::fs::create_dir_all(&patch).unwrap();
15091 }
15092 runner.state.save().unwrap();
15093 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15094 runner.state = RunState::load(&runner.state.id).unwrap();
15095 assert_eq!(runner.state.status, RunStatus::Blocked);
15096 assert!(runner.state.interrupted_implementations.contains(&'A'));
15097 assert_eq!(runner.state.candidates[0].commits, 0);
15098 assert!(!runner.state.candidates[0].empty);
15099 assert!(runner.state.candidates[0].failed.is_none());
15100 if failed_checkpoint {
15101 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15102 std::fs::remove_dir(patch).unwrap();
15103 }
15104 runner.state.resource_block = None;
15105 runner.state.config.disk.min_free_bytes = 0;
15106 runner.roles.implementers[0].command = vec![
15107 "sh".into(),
15108 "-c".into(),
15109 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15110 ];
15111 runner.implement().await.unwrap();
15112 assert!(runner.state.interrupted_implementations.is_empty());
15113 assert!(runner.state.candidate_preservation.is_empty());
15114 assert_eq!(runner.state.candidates[0].commits, 1);
15115 assert!(!runner.state.candidates[0].empty);
15116 assert!(
15117 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15118 .unwrap()
15119 .contains("retried empty interrupted seat")
15120 );
15121 if !solo {
15122 assert_eq!(runner.state.candidates[1].commits, 1);
15123 }
15124 }
15125 }
15126
15127 #[tokio::test]
15128 #[cfg(unix)]
15129 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15130 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15131 let (_tmp, mut runner) = preservation_resume_fixture().await;
15132 runner.state.candidates.retain(|c| c.label == 'A');
15133 runner.state.candidates[0].empty = false;
15134 runner.state.candidates[0].summary.clear();
15135 runner.state.tally = None;
15136 runner.state.judge_skipped = false;
15137 runner.state.status = RunStatus::Implementing;
15138 runner.state.config.disk.min_free_bytes = u64::MAX;
15141 let wt = runner.state.candidates[0].worktree.clone();
15142 std::fs::write(
15143 wt.join("interrupted-source"),
15144 "retain implementation work\n",
15145 )
15146 .unwrap();
15147 if committed {
15148 git::rescue_commit(&wt, "implementation checkpoint")
15149 .await
15150 .unwrap();
15151 }
15152 let lock = git::git(
15153 &wt,
15154 &[
15155 "rev-parse",
15156 "--path-format=absolute",
15157 "--git-path",
15158 "index.lock",
15159 ],
15160 )
15161 .await
15162 .unwrap();
15163 if block_rescue {
15164 std::fs::write(&lock, "fixture-owned lock").unwrap();
15165 }
15166 let mut implementer = spec("alpha");
15167 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15168 runner.roles.implementers = vec![implementer];
15169 runner.state.save().unwrap();
15170 let result = runner.implement().await;
15171 assert_eq!(result.is_err(), block_rescue);
15172 assert_eq!(
15173 runner.state.resource_block.as_ref().unwrap().kind,
15174 "disk pressure"
15175 );
15176 assert!(
15177 runner
15178 .state
15179 .events
15180 .iter()
15181 .any(|e| e.message.contains("agent execution interrupted"))
15182 );
15183 assert!(
15184 runner
15185 .state
15186 .dir()
15187 .join("artifacts/candidate-A-recovery/staged.patch")
15188 .exists()
15189 );
15190 if block_rescue {
15191 let saved = RunState::load(&runner.state.id).unwrap();
15192 assert!(saved.candidate_preservation.contains_key(&'A'));
15193 assert!(!saved.candidates[0].empty);
15194 assert_eq!(
15195 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15196 "retain implementation work\n"
15197 );
15198 std::fs::remove_file(lock).unwrap();
15199 runner.state = saved;
15200 } else {
15201 assert_eq!(runner.state.status, RunStatus::Blocked);
15202 assert_eq!(runner.state.candidates[0].commits, 1);
15203 }
15204 runner.roles.implementers.clear();
15207 runner.state.config.disk.min_free_bytes = 0;
15208 runner.state.resource_block = None;
15209 runner.implement().await.unwrap();
15210 assert!(runner.state.candidate_preservation.is_empty());
15211 assert_eq!(runner.state.candidates[0].commits, 1);
15212 assert!(!runner.state.candidates[0].empty);
15213 assert!(git::is_clean(&wt).await.unwrap());
15214 assert!(
15215 crate::run::read_artifact(&runner.state, "cand-A.patch")
15216 .unwrap()
15217 .contains("retain implementation work")
15218 );
15219 }
15220 }
15221
15222 #[tokio::test]
15223 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15224 let (_tmp, mut runner) = preservation_resume_fixture().await;
15225 runner.implement().await.unwrap();
15227 assert!(runner.state.candidate_preservation.is_empty());
15228 assert!(
15229 !runner
15230 .state
15231 .dir()
15232 .join("artifacts/candidate-A-recovery")
15233 .exists()
15234 );
15235 runner.fold_losers().await.unwrap();
15236 assert!(!runner.state.candidates[0].worktree.exists());
15237 for schema in [14, crate::run::SCHEMA] {
15238 runner.state.schema = schema;
15239 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15240 runner.state.save().unwrap();
15241 runner.state = RunState::load(&runner.state.id).unwrap();
15242 runner.implement().await.unwrap();
15243 assert!(runner.state.candidate_preservation.is_empty());
15244 assert!(runner.state.candidates[0].folded);
15245 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15246 }
15247 }
15248
15249 #[tokio::test]
15250 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15251 let (_tmp, mut runner) = preservation_resume_fixture().await;
15252 runner.state.candidates.retain(|c| c.label == 'A');
15253 runner.state.tally = None;
15254 runner.state.judge_skipped = false;
15255 let candidate = &mut runner.state.candidates[0];
15256 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15257 candidate.failed = Some("deployment validation unavailable".into());
15258 std::fs::write(
15259 candidate.worktree.join("repair"),
15260 "future launcher policy\n",
15261 )
15262 .unwrap();
15263 runner
15264 .state
15265 .candidate_preservation
15266 .insert('A', "awaiting persistence".into());
15267 runner.persist_candidates().await.unwrap();
15268 let candidate = &runner.state.candidates[0];
15269 assert!(!candidate.empty);
15270 assert_eq!(candidate.commits, 1);
15271 assert_eq!(candidate.files, 1);
15272 assert!(candidate.failed.is_none());
15273 assert!(candidate.summary.contains("EPERM"));
15274 assert!(
15275 crate::run::read_artifact(&runner.state, "cand-A.patch")
15276 .unwrap()
15277 .contains("future launcher policy")
15278 );
15279 assert!(runner.state.candidate_preservation.is_empty());
15280 }
15281
15282 #[tokio::test]
15283 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15284 let (_tmp, mut runner) = preservation_resume_fixture().await;
15285 let wt = runner.state.candidates[0].worktree.clone();
15286 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15287 runner.state.schema = 14;
15288 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15289 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15290 let tally = serde_json::to_value(&runner.state.tally).unwrap();
15291 for _ in 0..2 {
15292 assert!(runner.implement().await.is_err());
15293 assert_eq!(runner.state.status, RunStatus::Blocked);
15294 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15295 assert!(
15296 runner.state.candidates[0].empty,
15297 "old verdict must remain unusable"
15298 );
15299 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15300 assert!(runner.state.judge_skipped);
15301 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15302 assert!(
15303 crate::run::read_artifact(&runner.state, "cand-A.patch")
15304 .unwrap()
15305 .contains("owned edits")
15306 );
15307 assert!(wt.exists());
15308 assert_eq!(
15309 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15310 .await
15311 .unwrap(),
15312 1
15313 );
15314 runner.state = RunState::load(&runner.state.id).unwrap();
15315 }
15316 }
15317
15318 #[tokio::test]
15319 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15320 let (_tmp, mut runner) = preservation_resume_fixture().await;
15321 let wt = runner.state.candidates[0].worktree.clone();
15322 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15323 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15324 let index_lock = PathBuf::from(
15325 git::git(
15326 &wt,
15327 &[
15328 "rev-parse",
15329 "--path-format=absolute",
15330 "--git-path",
15331 "index.lock",
15332 ],
15333 )
15334 .await
15335 .unwrap(),
15336 );
15337 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15338 runner.state.schema = 14;
15339 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15340 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15341 assert!(runner.implement().await.is_err());
15342 assert!(runner.after_implement().is_err());
15343 assert!(runner.fold_losers().await.is_err());
15344 assert_eq!(
15345 std::fs::read_to_string(wt.join("recovered")).unwrap(),
15346 "failed seat edits"
15347 );
15348 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15349 runner.state = RunState::load(&runner.state.id).unwrap();
15350 std::fs::remove_file(index_lock).unwrap();
15351 assert!(runner.implement().await.is_err());
15353 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15354 assert!(runner.state.candidates[0].empty);
15355 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15356 assert!(
15357 crate::run::read_artifact(&runner.state, "cand-A.patch")
15358 .unwrap()
15359 .contains("failed seat edits")
15360 );
15361 assert!(wt.exists());
15362 }
15363
15364 #[tokio::test]
15365 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
15366 let (_tmp, mut runner) = preservation_resume_fixture().await;
15367 let repo = runner.state.repo.clone();
15368 let common = git::git(
15369 &repo,
15370 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
15371 )
15372 .await
15373 .unwrap();
15374 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
15375 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
15376 assert!(runner.fold_losers().await.is_err());
15377 assert!(!runner.state.candidates[0].worktree.exists());
15378 assert!(!runner.state.candidates[0].folded);
15379 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
15380 runner.state = RunState::load(&runner.state.id).unwrap();
15381 std::fs::remove_file(branch_lock).unwrap();
15382 let judge = runner.state.worktree_root().join("judge-1");
15383 git::worktree_add_detached(&repo, &judge, "main")
15384 .await
15385 .unwrap();
15386 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
15387 runner.roles.judges = vec![spec("judge")];
15388 assert!(runner.fold_losers().await.is_err());
15389 runner.state = RunState::load(&runner.state.id).unwrap();
15390 assert!(
15391 runner.state.candidates[0].folded,
15392 "candidate progress saved before judge failure"
15393 );
15394 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15395 assert!(judge.join("uncommitted").exists());
15396 std::fs::remove_file(judge.join("uncommitted")).unwrap();
15397 runner.fold_losers().await.unwrap();
15398 assert!(!judge.exists());
15399 runner.fold_losers().await.unwrap();
15400 }
15401
15402 #[tokio::test]
15403 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15404 let (_tmp, mut runner) = preservation_resume_fixture().await;
15405 runner.state.candidates.retain(|c| c.label == 'B');
15406 runner.state.tally = None;
15407 runner.state.judge_skipped = false;
15408 runner.state.prep_complete = Some(true);
15409 runner.state.config.graph.advise = false;
15410 runner.state.config.graph.selected_implementer = Some("alpha".into());
15411 let wt = runner.state.candidates[0].worktree.clone();
15412 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15413 runner
15414 .state
15415 .candidate_preservation
15416 .insert('B', "awaiting persistence".into());
15417 let index_lock = git::git(
15418 &wt,
15419 &[
15420 "rev-parse",
15421 "--path-format=absolute",
15422 "--git-path",
15423 "index.lock",
15424 ],
15425 )
15426 .await
15427 .unwrap();
15428 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15429 runner.state.save().unwrap();
15430 assert!(runner.execute_graph().await.is_err());
15431 assert_eq!(runner.state.status, RunStatus::Blocked);
15432 assert!(runner.state.candidate_preservation.contains_key(&'B'));
15433 std::fs::remove_file(index_lock).unwrap();
15434 for _ in 0..2 {
15437 runner.state = RunState::load(&runner.state.id).unwrap();
15438 runner.pause.park();
15439 runner.execute_graph().await.unwrap();
15440 assert_eq!(runner.state.status, RunStatus::Implementing);
15441 assert!(runner.state.candidate_preservation.is_empty());
15442 assert_eq!(runner.state.candidates[0].commits, 2);
15443 assert!(runner.state.tally.is_none());
15444 }
15445 runner.implement().await.unwrap();
15446 assert_eq!(runner.state.status, RunStatus::Judging);
15447 runner.state.status = RunStatus::Blocked;
15449 runner.state.candidates[0].failed = Some("implementation failed".into());
15450 runner.state.save().unwrap();
15451 let error = runner.execute_graph().await.unwrap_err();
15452 assert!(
15453 error
15454 .to_string()
15455 .contains("selected implementation is held")
15456 );
15457 }
15458
15459 #[tokio::test]
15460 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15461 let (_tmp, mut runner) = preservation_resume_fixture().await;
15462 let repo = runner.state.repo.clone();
15463 let candidate = runner.state.candidates[0].clone();
15464 let registered = git::worktree_holding(&repo, &candidate.branch)
15467 .await
15468 .unwrap()
15469 .unwrap();
15470 assert_eq!(
15471 registered.canonicalize().unwrap(),
15472 candidate.worktree.canonicalize().unwrap()
15473 );
15474 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15475 assert_eq!(
15476 git::worktree_holding(&repo, &candidate.branch)
15477 .await
15478 .unwrap(),
15479 Some(registered)
15480 );
15481 assert!(
15482 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15483 "the stale registration initially keeps the branch checked out"
15484 );
15485 runner.fold_losers().await.unwrap();
15486 assert!(runner.state.candidates[0].folded);
15487 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15488 assert_eq!(
15489 git::worktree_holding(&repo, &candidate.branch)
15490 .await
15491 .unwrap(),
15492 None
15493 );
15494 runner.state = RunState::load(&runner.state.id).unwrap();
15495 runner.fold_losers().await.unwrap();
15496 }
15497
15498 #[tokio::test]
15499 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15500 let (_tmp, mut runner) = preservation_resume_fixture().await;
15501 let repo = runner.state.repo.clone();
15502 let candidate = runner.state.candidates[0].clone();
15503 let winner = runner.state.candidates[1].clone();
15504 let registered = git::worktree_holding(&repo, &candidate.branch)
15505 .await
15506 .unwrap()
15507 .unwrap();
15508 assert_eq!(
15509 registered.canonicalize().unwrap(),
15510 candidate.worktree.canonicalize().unwrap()
15511 );
15512 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15513 assert_eq!(
15514 git::worktree_holding(&repo, &candidate.branch)
15515 .await
15516 .unwrap(),
15517 Some(registered)
15518 );
15519 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15520
15521 for _ in 0..2 {
15522 fold_run(&mut runner.state, false, &crate::run::home())
15523 .await
15524 .unwrap();
15525 runner.state = RunState::load(&runner.state.id).unwrap();
15526 assert!(runner.state.candidates[0].folded);
15527 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15528 assert_eq!(
15529 git::worktree_holding(&repo, &candidate.branch)
15530 .await
15531 .unwrap(),
15532 None
15533 );
15534 assert!(!runner.state.candidates[1].folded);
15535 assert!(winner.worktree.exists());
15536 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15537 }
15538 }
15539
15540 #[tokio::test]
15541 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15542 ask_test_home();
15543 let tmp = tempfile::tempdir().unwrap();
15544 let repo = tmp.path().join("repo");
15545 std::fs::create_dir(&repo).unwrap();
15546 init_repo(&repo);
15547 let mut runner = runner_at(RunStatus::Implementing);
15548 runner.state.repo = repo.clone();
15549 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15550 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15551 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15552 let branch = format!("candidate-{label}");
15553 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15554 git::worktree_add_branch(&repo, &wt, &branch, "main")
15555 .await
15556 .unwrap();
15557 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15558 git::git(&wt, &["add", "README.md"]).await.unwrap();
15559 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15560 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15561 runner.state.candidates.push(Candidate {
15562 index,
15563 label,
15564 agent: "alpha".to_owned(),
15565 branch,
15566 worktree: wt,
15567 summary: "completed implementation".to_owned(),
15568 stat: String::new(),
15569 files: 0,
15570 commits: 0,
15571 empty: false,
15572 failed: None,
15573 verified_noop: None,
15574 duration_ms: 42,
15575 folded: false,
15576 });
15577 runner
15578 .state
15579 .candidate_preservation
15580 .insert(label, "awaiting persistence".to_owned());
15581 }
15582 let wt = runner.state.candidates[0].worktree.clone();
15583 let index_lock = git::git(
15584 &wt,
15585 &[
15586 "rev-parse",
15587 "--path-format=absolute",
15588 "--git-path",
15589 "index.lock",
15590 ],
15591 )
15592 .await
15593 .unwrap();
15594 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15595 runner.state.save().unwrap();
15596 assert!(runner.persist_candidates().await.is_err());
15597 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15598 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15599 assert_eq!(
15600 runner.state.candidates[1].commits, 1,
15601 "other results are still collected"
15602 );
15603 assert!(
15604 !runner.state.candidates[0].empty,
15605 "failed rescue is not no change"
15606 );
15607 assert_eq!(
15608 runner.state.candidates[0].summary,
15609 "completed implementation"
15610 );
15611 assert!(runner.after_implement().is_err());
15612 let home = tmp.path().join("fold-home");
15613 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15614 assert!(wt.join("new-file").exists());
15615 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15616 assert!(recovery.join("staged.patch").exists());
15617 assert!(recovery.join("unstaged.patch").exists());
15618 assert_eq!(
15619 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15620 "untracked edits\n"
15621 );
15622 runner.state.begin_pass();
15623 runner.state.end_pass_parked(ParkCause::Upgrade {
15624 from: "1.0".into(),
15625 to: Some("1.1".into()),
15626 });
15627 runner.state.save().unwrap();
15628 runner.state = RunState::load(&runner.state.id).unwrap();
15629 assert!(matches!(
15630 runner.state.passes.last().unwrap().end,
15631 Some(crate::run::PassEnd::Parked {
15632 cause: ParkCause::Upgrade { .. }
15633 })
15634 ));
15635 std::fs::remove_file(index_lock).unwrap();
15636 runner.implement().await.unwrap();
15638 assert!(runner.state.candidate_preservation.is_empty());
15639 assert!(
15640 runner
15641 .state
15642 .candidates
15643 .iter()
15644 .all(|c| c.commits == 1 && !c.empty)
15645 );
15646 assert!(git::is_clean(&wt).await.unwrap());
15647 assert!(
15648 crate::run::read_artifact(&runner.state, "cand-A.patch")
15649 .unwrap()
15650 .contains("later tracked edits")
15651 );
15652 }
15653
15654 #[tokio::test]
15655 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15656 ask_test_home();
15657 let tmp = tempfile::tempdir().unwrap();
15658 let repo = tmp.path().join("repo");
15659 std::fs::create_dir(&repo).unwrap();
15660 init_repo(&repo);
15661 let wt = tmp.path().join("candidate");
15662 git::worktree_add_branch(&repo, &wt, "candidate", "main")
15663 .await
15664 .unwrap();
15665 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15666 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15667 assert!(wt.join("uncommitted").exists());
15668 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15669 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15670 let mut runner = runner_at(RunStatus::Implementing);
15671 runner.state.repo = repo.clone();
15672 runner.state.base_commit = "missing-base".to_owned();
15673 runner.state.candidates.push(Candidate {
15674 index: 0,
15675 label: 'A',
15676 agent: "alpha".into(),
15677 branch: "candidate".into(),
15678 worktree: wt.clone(),
15679 summary: "completed".into(),
15680 stat: String::new(),
15681 files: 0,
15682 commits: 0,
15683 empty: false,
15684 failed: None,
15685 verified_noop: None,
15686 duration_ms: 1,
15687 folded: false,
15688 });
15689 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15690 assert!(
15691 fold_run(&mut runner.state, true, &tmp.path().join("home"))
15692 .await
15693 .is_err()
15694 );
15695 assert!(!runner.state.candidates[0].folded);
15696 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15697 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15698 runner
15699 .state
15700 .candidate_preservation
15701 .insert('A', "pending".into());
15702 assert!(runner.persist_candidates().await.is_err());
15703 assert!(!runner.state.candidates[0].empty);
15704 assert!(!runner.state.candidates[0].folded);
15705 assert!(wt.exists());
15706 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15707 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15708 std::fs::create_dir_all(&patch_path).unwrap();
15709 assert!(
15710 runner.persist_candidates().await.is_err(),
15711 "artifact failure stays pending"
15712 );
15713 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15714 assert!(!runner.state.candidates[0].empty);
15715 std::fs::remove_dir(patch_path).unwrap();
15716 runner.persist_candidates().await.unwrap();
15717 assert!(
15718 runner.state.candidates[0].empty,
15719 "only successful inspection confirms no change"
15720 );
15721 }
15722
15723 fn init_repo(dir: &Path) {
15724 let run = |args: &[&str]| {
15725 let out = std::process::Command::new("git")
15726 .args(args)
15727 .current_dir(dir)
15728 .quiet()
15729 .output()
15730 .expect("spawn git");
15731 assert!(
15732 out.status.success(),
15733 "git {args:?} failed: {}",
15734 String::from_utf8_lossy(&out.stderr)
15735 );
15736 };
15737 run(&["init", "-b", "main"]);
15738 run(&["config", "user.name", "magi test"]);
15739 run(&["config", "user.email", "magi@example.com"]);
15740 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15741 run(&["add", "-A"]);
15742 run(&["commit", "-m", "init"]);
15743 }
15744
15745 fn ask_test_home() {
15753 crate::run::pin_test_home();
15754 }
15755
15756 #[test]
15757 fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
15758 ask_test_home();
15759 let mut runner = runner_at(RunStatus::Prep);
15760 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
15761 std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
15764 let error = guard
15765 .finish(
15766 &mut runner.state,
15767 Err(
15768 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
15769 .into(),
15770 ),
15771 )
15772 .unwrap_err();
15773 assert_eq!(
15774 error.downcast_ref::<std::io::Error>().unwrap().kind(),
15775 std::io::ErrorKind::PermissionDenied
15776 );
15777 assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
15778 let message = format!("{error:#}");
15779 assert!(message.contains("original refusal"));
15780 assert!(!message.contains("could not save terminal"));
15781 assert_eq!(
15782 error.chain().count(),
15783 2,
15784 "only startup attribution wraps the original error"
15785 );
15786 let saved = RunState::load(&runner.state.id).unwrap();
15787 assert_eq!(saved.status, RunStatus::Blocked);
15788 assert!(!saved.startup.unwrap().complete);
15789 }
15790
15791 #[test]
15792 fn reserved_startup_interruption_is_blocked_and_not_resumable() {
15793 ask_test_home();
15794 let mut runner = runner_at(RunStatus::Prep);
15795 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
15796 let id = runner.state.id.clone();
15797 drop(guard);
15798 let saved = RunState::load(&id).unwrap();
15799 assert_eq!(saved.status, RunStatus::Blocked);
15800 assert!(!saved.startup.as_ref().unwrap().complete);
15801 assert!(
15802 saved
15803 .startup
15804 .unwrap()
15805 .failure
15806 .unwrap()
15807 .contains("interrupted")
15808 );
15809 assert!(Runner::resume(&id).is_err());
15810 }
15811
15812 #[test]
15813 fn startup_failure_keeps_typed_cause_and_separate_task_history() {
15814 ask_test_home();
15815 let mut runner = runner_at(RunStatus::Prep);
15816 let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
15817 let error = guard
15818 .finish(
15819 &mut runner.state,
15820 Err(
15821 std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
15822 .into(),
15823 ),
15824 )
15825 .unwrap_err();
15826 assert_eq!(
15827 error.downcast_ref::<std::io::Error>().unwrap().kind(),
15828 std::io::ErrorKind::PermissionDenied
15829 );
15830 let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
15831 let mut task = crate::queue::Task::new(
15832 "review".into(),
15833 "criteria".into(),
15834 runner.state.repo.clone(),
15835 crate::queue::Source::Human,
15836 );
15837 task.record_startup_failure(failure.clone());
15838 task.record_startup_failure(failure.clone());
15839 assert_eq!(task.startup_failures.len(), 1);
15840 assert!(task.runs.is_empty());
15841 assert_eq!(task.attempts, 0);
15842 }
15843
15844 #[cfg(unix)]
15845 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15846 ask_test_home();
15847 let tmp = tempfile::tempdir().unwrap();
15848 let repo = tmp.path().join("repo");
15849 std::fs::create_dir(&repo).unwrap();
15850 init_repo(&repo);
15851 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15852 let origin = tmp.path().join("origin.git");
15853 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15854 .await
15855 .unwrap();
15856 git::git(
15857 &repo,
15858 &["remote", "add", "origin", origin.to_str().unwrap()],
15859 )
15860 .await
15861 .unwrap();
15862 git::git(&repo, &["checkout", "-b", "fix/retained"])
15863 .await
15864 .unwrap();
15865 std::fs::write(repo.join("retained"), "retained change").unwrap();
15866 git::git(&repo, &["add", "."]).await.unwrap();
15867 git::git(&repo, &["commit", "-m", "retained implementation"])
15868 .await
15869 .unwrap();
15870 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15871 git::git(&repo, &["checkout", "main"]).await.unwrap();
15872 std::fs::write(repo.join("base-change"), "new base").unwrap();
15873 git::git(&repo, &["add", "."]).await.unwrap();
15874 git::git(&repo, &["commit", "-m", "base moved"])
15875 .await
15876 .unwrap();
15877 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15878 git::git(&repo, &["checkout", "fix/retained"])
15879 .await
15880 .unwrap();
15881 let mut runner = runner_at(RunStatus::Voting);
15882 runner.state.repo = repo.clone();
15883 runner.state.base_commit = base;
15884 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15885 runner.state.config.disk.min_free_bytes = 0;
15886 runner.state.config.graph.review_rounds = 1;
15887 runner.state.config.verify.e2e = vec!["true".into()];
15888 runner.state.config.verify.gate = vec!["true".into()];
15889 runner.state.candidates = vec![Candidate {
15890 index: 0,
15891 label: 'A',
15892 agent: "fixture".into(),
15893 branch: "fix/retained".into(),
15894 worktree: repo.clone(),
15895 summary: String::new(),
15896 stat: String::new(),
15897 files: 1,
15898 commits: 1,
15899 empty: false,
15900 failed: None,
15901 verified_noop: None,
15902 duration_ms: 0,
15903 folded: false,
15904 }];
15905 runner.state.tally = Some(Tally {
15906 first_choice: BTreeMap::new(),
15907 borda: BTreeMap::new(),
15908 winner: 'A',
15909 rankings: 0,
15910 unanimous_initial: true,
15911 deliberated: false,
15912 changed_votes: 0,
15913 unanimous_final: true,
15914 tie_break: None,
15915 judges: 0,
15916 present: 0,
15917 quorum: 0,
15918 met_quorum: true,
15919 uncontested: Some("review-only".into()),
15920 });
15921 let mut task = crate::queue::Task::new(
15922 "fixture".into(),
15923 "fixture".into(),
15924 repo,
15925 crate::queue::Source::Human,
15926 );
15927 task.link_run(&runner.state.id);
15928 crate::queue::Queue::open().put(&mut task).unwrap();
15929 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15930 runner.state.base_sync = Some(BaseSync {
15931 tip: String::new(),
15932 behind: 1,
15933 attempts: BASE_SYNC_ROUNDS,
15934 conflict: None,
15935 already_in: None,
15936 });
15937 runner.sync_to_base().await.unwrap();
15938 assert_eq!(runner.state.status, RunStatus::Blocked);
15939 (tmp, runner, head, task.id)
15940 }
15941
15942 #[cfg(unix)]
15943 #[tokio::test]
15944 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15945 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15946 if std::env::var_os(CHILD).is_none() {
15947 let home = tempfile::tempdir().unwrap();
15948 let output = std::process::Command::new(std::env::current_exe().unwrap())
15949 .args([
15950 "--exact",
15951 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15952 "--nocapture",
15953 ])
15954 .env(CHILD, "1")
15955 .env("MAGI_HOME", home.path())
15956 .env("MAGI_WORKER_HOME", home.path())
15957 .env("MAGI_WORKER_ID", "abcdef")
15958 .output()
15959 .unwrap();
15960 assert!(
15961 output.status.success()
15962 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15963 "{}\n{}",
15964 String::from_utf8_lossy(&output.stdout),
15965 String::from_utf8_lossy(&output.stderr)
15966 );
15967 return;
15968 }
15969 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15970 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15971 let home = crate::run::home();
15972 let queue = crate::queue::Queue::open();
15973 let mut owner = queue.get(&task).unwrap();
15974 owner.release_operator("test-release", None);
15975 owner.start(runner.state.id.clone());
15976 queue.put(&mut owner).unwrap();
15977 runner.state.driver_exited = true;
15978 runner.state.task_instruction = Some(owner.instruction.clone());
15979 runner.state.save().unwrap();
15980 let _claim = queue.claim(&task).unwrap();
15981 let reserve = |id: &str| {
15982 let directory = home.join("workers/jobs").join(id);
15983 std::fs::create_dir_all(&directory).unwrap();
15984 let job = crate::worker::Job {
15985 id: id.into(),
15986 sequence: 1,
15987 kind: "task".into(),
15988 key: format!("task:{task}"),
15989 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15990 generation: "fixture-generation".into(),
15991 version: None,
15992 status: crate::worker::Status::Running,
15993 result: None,
15994 pid: Some(std::process::id()),
15995 created_at: 0,
15996 finished_at: None,
15997 };
15998 std::fs::write(
15999 directory.join("job.json"),
16000 serde_json::to_vec(&job).unwrap(),
16001 )
16002 .unwrap();
16003 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16004 };
16005 let _own = reserve("abcdef");
16006 let other = reserve("fedcba");
16007 assert!(
16008 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16009 .await
16010 .is_err()
16011 );
16012 assert!(
16013 RunState::load(&runner.state.id)
16014 .unwrap()
16015 .base_sync_recovery
16016 .authorizations
16017 .is_empty()
16018 );
16019 drop(other);
16020 assert!(crate::daemon::is_working_on(
16021 &home,
16022 &runner.state.id,
16023 Timestamp::now()
16024 ));
16025 assert!(
16026 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16027 );
16028 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16029 .await
16030 .unwrap();
16031 assert!(
16032 RunState::load(&runner.state.id)
16033 .unwrap()
16034 .base_recovery_pending()
16035 );
16036 }
16037
16038 #[cfg(unix)]
16039 #[tokio::test]
16040 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16041 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16042 let home = crate::run::home();
16043 runner.state.driver_pid = Some(std::process::id());
16044 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16045 runner.state.driver_exited = false;
16046 runner.state.begin_pass();
16047 runner.state.save().unwrap();
16048 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16049 driver.record(&runner.state).unwrap();
16050 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16051 drop(driver);
16053 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16054 assert_eq!(
16055 runner.state.liveness_under(&home, false),
16056 crate::run::Liveness::Dead
16057 );
16058 let queue = crate::queue::Queue::open();
16059 let _claim = queue.claim(&task).unwrap();
16060 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16061 .await
16062 .unwrap();
16063 let disk = RunState::load(&runner.state.id).unwrap();
16064 assert!(!disk.driver_exited);
16065 assert!(disk.base_recovery_pending());
16066 }
16067
16068 #[cfg(unix)]
16069 #[tokio::test]
16070 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16071 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16072 if std::env::var_os(CHILD).is_none() {
16073 use std::os::unix::fs::PermissionsExt;
16074 let tmp = tempfile::tempdir().unwrap();
16075 let gh = tmp.path().join("gh");
16076 std::fs::write(&gh, r#"#!/bin/sh
16077case "$1 $2" in
16078 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16079 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16080 *) exit 2 ;;
16081esac
16082"#).unwrap();
16083 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16084 let mut paths = vec![tmp.path().to_path_buf()];
16085 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16086 let output = std::process::Command::new(std::env::current_exe().unwrap())
16087 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16088 .env(CHILD, "1")
16089 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16090 .env("PATH", std::env::join_paths(paths).unwrap())
16091 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16092 .output().unwrap();
16093 assert!(
16094 output.status.success()
16095 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16096 "{}\n{}",
16097 String::from_utf8_lossy(&output.stdout),
16098 String::from_utf8_lossy(&output.stderr)
16099 );
16100 return;
16101 }
16102 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16103 let repo = runner.state.repo.clone();
16104 git::git(&repo, &["checkout", "main"]).await.unwrap();
16105 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16106 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16107 git::git(&repo, &["checkout", "fix/retained"])
16108 .await
16109 .unwrap();
16110 runner.state.pr = Some(crate::run::PrRecord {
16111 url: "https://example.test/project/pull/1".into(),
16112 number: 1,
16113 state: "open".into(),
16114 checks: "unknown".into(),
16115 round: 0,
16116 rounds: 1,
16117 red_at_merge: Vec::new(),
16118 });
16119 runner.sync_to_base().await.unwrap();
16120 assert_eq!(runner.state.status, RunStatus::Blocked);
16121 assert_eq!(
16122 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16123 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16124 );
16125 assert!(runner.state.base_sync_closure_pending());
16126 assert!(crate::handover::can_resume(&runner.state));
16127 let queue = crate::queue::Queue::open();
16128 assert!(!crate::base_sync_recovery::held_task(
16129 &queue,
16130 &queue.get(&task).unwrap()
16131 ));
16132 assert!(runner.state.authorize_base_sync(&task).is_err());
16133 runner.state.base_sync_recovery.stop = None;
16135 runner.state.status = RunStatus::Voting;
16136 runner.state.gate_ran = true;
16137 runner.state.config.graph.review_rounds = 0;
16138 runner.state.save().unwrap();
16139 assert!(crate::handover::can_resume(&runner.state));
16140 runner.state = RunState::load(&runner.state.id).unwrap();
16141 runner.execute_graph().await.unwrap();
16142 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16143 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16144 assert_eq!(
16145 runner.state.base_sync.as_ref().unwrap().attempts,
16146 BASE_SYNC_ROUNDS
16147 );
16148 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16149 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16150 assert!(runner.state.reviews.is_empty());
16151 assert!(runner.state.gate.is_empty());
16152 }
16153
16154 #[cfg(unix)]
16155 #[tokio::test]
16156 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16157 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16158 runner.state.driver_exited = true;
16159 runner.state.save().unwrap();
16160 let home = crate::run::home();
16161 let queue = crate::queue::Queue::open();
16162 let _claim = queue.claim(&task).unwrap();
16163 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16164 assert!(
16165 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16166 .await
16167 .is_err()
16168 );
16169 drop(lease);
16170 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16171 assert!(
16172 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16173 .await
16174 .is_err()
16175 );
16176 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16177 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16178 .await
16179 .unwrap();
16180 let disk = RunState::load(&runner.state.id).unwrap();
16181 assert!(disk.base_recovery_pending());
16182 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16183 assert!(
16184 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16185 .await
16186 .is_err()
16187 );
16188 assert_eq!(
16189 RunState::load(&runner.state.id)
16190 .unwrap()
16191 .base_sync_recovery
16192 .authorizations
16193 .len(),
16194 1
16195 );
16196 let mut owner = queue.get(&task).unwrap();
16197 owner.runs.push("20260101-000000-ffff".into());
16198 queue.put(&mut owner).unwrap();
16199 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16200 .await
16201 .unwrap_err();
16202 assert!(error.to_string().contains("run task ownership changed"));
16203 }
16204
16205 #[cfg(unix)]
16206 #[tokio::test]
16207 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16208 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16209 let mut legacy = serde_json::to_value(&runner.state).unwrap();
16210 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16211 legacy["status"] = serde_json::json!("voting");
16212 runner.state = serde_json::from_value(legacy).unwrap();
16213 runner.state.save().unwrap();
16214 for _ in 0..3 {
16215 runner.execute_graph().await.unwrap();
16216 assert_eq!(runner.state.status, RunStatus::Blocked);
16217 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16218 assert!(!crate::handover::can_resume(&runner.state));
16219 assert_eq!(
16220 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16221 head
16222 );
16223 assert!(runner.state.reviews.is_empty());
16224 assert!(!runner.state.gate_ran);
16225 }
16226 let notice = crate::notices::run_ended(&runner.state).unwrap();
16227 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16228 runner.state.begin_pass();
16229 runner.state.status = RunStatus::Voting;
16230 runner.deliberate().await.unwrap();
16231 runner.vote().await.unwrap();
16232 runner.finish_after_tally().await.unwrap();
16233 assert_eq!(runner.state.status, RunStatus::Blocked);
16234 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16235 assert_eq!(
16236 crate::notices::run_ended(&runner.state).unwrap().since,
16237 notice.since
16238 );
16239 }
16240
16241 #[cfg(unix)]
16242 #[tokio::test]
16243 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16244 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16245 for detail in [
16246 "fix/retained conflicts with origin/main and did not rebase: conflict",
16247 "rebased fix/retained locally but origin refused the push",
16248 "unknown failure",
16249 ] {
16250 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16251 runner.state.record_base_sync_stop(&head);
16252 assert!(runner.state.authorize_base_sync(&task).is_err());
16253 runner.state.status = RunStatus::Voting;
16254 runner.execute_graph().await.unwrap();
16255 assert_eq!(runner.state.status, RunStatus::Blocked);
16256 assert!(!crate::handover::can_resume(&runner.state));
16257 assert_eq!(
16258 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16259 head
16260 );
16261 }
16262 }
16263
16264 #[cfg(unix)]
16265 #[tokio::test]
16266 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16267 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16268 runner.state.authorize_base_sync(&task).unwrap();
16269 assert!(runner.state.consume_base_recovery(&head));
16270 runner.state.save().unwrap();
16271 runner.state = RunState::load(&runner.state.id).unwrap();
16272 runner.execute_graph().await.unwrap();
16273 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16274 assert_eq!(runner.state.status, RunStatus::Blocked);
16275 assert!(!runner.state.consume_base_recovery(&head));
16276 assert!(!crate::handover::can_resume(&runner.state));
16277 runner.state.authorize_base_sync(&task).unwrap();
16278 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
16279 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16280 assert_eq!(
16281 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16282 head
16283 );
16284 }
16285
16286 #[cfg(unix)]
16287 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
16288 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
16289 runner.state.prep_complete = Some(true);
16291 runner.state.base_sync = None;
16292 runner.state.base_sync_recovery = Default::default();
16293 runner.state.config.graph.gate_fix_rounds = 1;
16294 let mut review = review_round(true, 0, 1, 1, false, true);
16295 review.head = head.clone();
16296 review.verified_head = Some(head.clone());
16297 runner.state.reviews = vec![review];
16298 runner.state.gate = vec![CommandOutcome {
16299 command: "old gate".into(),
16300 code: Some(1),
16301 output_tail: "old failure".into(),
16302 duration_ms: 0,
16303 resource_blocked: false,
16304 }];
16305 runner.state.gate_head = Some(head.clone());
16306 runner.state.gate_ran = true;
16307 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
16308 agent: "fixture".into(),
16309 failed: runner.state.gate.clone(),
16310 notes: "spent".into(),
16311 committed: false,
16312 error: None,
16313 }];
16314 runner.state.land_approval = Some(crate::run::LandApproval {
16315 question: "old approval".into(),
16316 head: head.clone(),
16317 });
16318 runner.state.status = RunStatus::Blocked;
16319 (tmp, runner, head)
16320 }
16321
16322 #[cfg(unix)]
16323 #[tokio::test]
16324 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
16325 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
16326 let (tmp, mut runner, old) = resumed_gate_fixture().await;
16327 let phases = tmp.path().join("phases");
16328 runner.state.config.verify.e2e = vec![format!(
16329 "echo e2e >> '{}'; {}",
16330 phases.display(),
16331 if e2e_ok {
16332 "true"
16333 } else {
16334 "echo new-e2e-failure; false"
16335 }
16336 )];
16337 runner.state.config.verify.gate = vec![format!(
16338 "echo gate >> '{}'; {}",
16339 phases.display(),
16340 if gate_ok {
16341 "true"
16342 } else {
16343 "echo new-gate-failure; false"
16344 }
16345 )];
16346 runner.state.save().unwrap();
16347 runner.state = RunState::load(&runner.state.id).unwrap();
16348 runner.sync_to_base().await.unwrap();
16349 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16350 assert_ne!(head, old);
16351 runner.state = RunState::load(&runner.state.id).unwrap();
16353 runner.execute_graph().await.unwrap();
16354 assert_eq!(runner.state.gate_fixes.len(), 1);
16355 assert_eq!(runner.state.reviews.len(), 1);
16356 assert_eq!(runner.state.reviews[0].head, old);
16357 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
16358 assert_eq!(runner.state.verification_history.len(), 1);
16359 let history = &runner.state.verification_history[0];
16360 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
16361 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16362 assert!(history.archived_at.is_some());
16363 assert_eq!(history.fixes_spent, Some(1));
16364 assert_eq!(history.gate[0].output_tail, "old failure");
16365 let archive_events: Vec<_> = runner
16366 .state
16367 .events
16368 .iter()
16369 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
16370 .collect();
16371 assert_eq!(archive_events.len(), 1);
16372 assert!(archive_events[0].message.contains(&short(&old)));
16373 assert!(archive_events[0].message.contains(&short(&head)));
16374 assert!(
16375 archive_events[0]
16376 .message
16377 .contains("1 gate-fix round(s) already spent stay spent")
16378 );
16379 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
16380 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
16381 if e2e_ok {
16382 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16383 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
16384 } else {
16385 assert!(!runner.state.gate_ran);
16386 }
16387 assert_eq!(
16388 runner.state.status,
16389 if e2e_ok && gate_ok {
16390 RunStatus::Ready
16391 } else {
16392 RunStatus::Blocked
16393 }
16394 );
16395 if !e2e_ok || !gate_ok {
16396 let cause = runner.state.verification_blocking_cause().unwrap();
16397 assert!(cause.contains(&head));
16398 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
16399 runner.state.event("land", "rebased successfully");
16401 let notice = crate::notices::run_ended(&runner.state).unwrap();
16402 assert!(!format!("{notice:?}").contains("rebased successfully"));
16403 assert!(runner.state.merge.is_none());
16404 }
16405 let before = std::fs::read_to_string(&phases).unwrap();
16406 runner.state.save().unwrap();
16407 runner.state = RunState::load(&runner.state.id).unwrap();
16408 runner.review_loop().await.unwrap();
16409 runner.gate().await.unwrap();
16410 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
16411 assert_eq!(runner.state.verification_history.len(), 1);
16412 let history = &runner.state.verification_history[0];
16413 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16414 assert!(history.archived_at.is_some());
16415 assert_eq!(history.fixes_spent, Some(1));
16416 }
16417 }
16418
16419 #[cfg(unix)]
16420 #[tokio::test]
16421 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
16422 let (tmp, mut runner, head) = resumed_gate_fixture().await;
16423 let marker = tmp.path().join("gate-ran");
16424 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
16425 runner.review_loop().await.unwrap();
16426 runner.gate().await.unwrap();
16427 assert_eq!(runner.state.status, RunStatus::Blocked);
16428 assert!(!marker.exists());
16429 assert!(runner.state.verification_history.is_empty());
16430 runner.state.gate_head = None;
16431 runner.gate().await.unwrap();
16432 assert!(marker.exists());
16433 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16434 assert_eq!(runner.state.verification_history.len(), 1);
16435 assert_eq!(runner.state.gate_fixes.len(), 1);
16436 }
16437
16438 #[cfg(unix)]
16439 #[tokio::test]
16440 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
16441 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16442 runner.sync_to_base().await.unwrap();
16443 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16444 runner.state.reconcile_verification_head(&head);
16445 for outcomes in [
16447 Vec::new(),
16448 vec![CommandOutcome {
16449 command: "true".into(),
16450 code: None,
16451 output_tail: "cache unavailable".into(),
16452 duration_ms: 0,
16453 resource_blocked: true,
16454 }],
16455 ] {
16456 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16457 head: head.clone(),
16458 passed: false,
16459 outcomes,
16460 });
16461 runner.state.gate_ran = false;
16462 runner.state.gate.clear();
16463 runner.state.save().unwrap();
16464 runner.state = RunState::load(&runner.state.id).unwrap();
16465 runner.merge().await.unwrap();
16466 assert!(runner.state.merge.is_none());
16467 runner.review_loop().await.unwrap();
16468 runner.gate().await.unwrap();
16469 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
16470 assert!(runner.state.gate_allows_progress());
16471 assert_eq!(runner.state.gate_fixes.len(), 1);
16472 assert!(runner.state.merge.is_none());
16473 }
16474 git::git(
16476 &runner.state.repo,
16477 &["commit", "--allow-empty", "-m", "external update"],
16478 )
16479 .await
16480 .unwrap();
16481 runner.merge().await.unwrap();
16482 assert!(runner.state.merge.is_none());
16483 assert_eq!(runner.state.status, RunStatus::Blocked);
16484 }
16485
16486 #[cfg(unix)]
16487 #[tokio::test]
16488 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16489 for waiting_on_gate in [false, true] {
16490 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16491 runner.sync_to_base().await.unwrap();
16492 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16493 runner.state.reconcile_verification_head(&head);
16494 if waiting_on_gate {
16495 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16496 head: head.clone(),
16497 passed: true,
16498 outcomes: Vec::new(),
16499 });
16500 }
16501 let cache = tmp.path().join("cache");
16502 runner.state.config.verify.e2e =
16503 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16504 runner.state.config.verify.gate = vec!["true".into()];
16505 runner.state.config.graph.timeout_resource_wait = 1;
16506 let owner =
16507 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16508 let guard =
16509 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16510 crate::cache::AcquireOutcome::Acquired(g) => g,
16511 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16512 };
16513 runner.review_loop().await.unwrap();
16514 runner.gate().await.unwrap();
16515 assert!(!runner.state.gate_ran);
16516 assert_eq!(runner.state.gate_fixes.len(), 1);
16517 assert!(runner.state.resource_block.is_some());
16518 runner.merge().await.unwrap();
16519 assert!(runner.state.merge.is_none());
16520 drop(guard);
16521 runner.state = RunState::load(&runner.state.id).unwrap();
16522 runner.state.resource_block = None;
16524 runner.review_loop().await.unwrap();
16525 runner.gate().await.unwrap();
16526 assert!(runner.state.gate_allows_progress());
16527 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16528 assert_eq!(runner.state.gate_fixes.len(), 1);
16529 }
16530 }
16531
16532 #[cfg(unix)]
16533 #[tokio::test]
16534 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16535 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16536 runner.sync_to_base().await.unwrap();
16537 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16538 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16539 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16541 runner.review_loop().await.unwrap();
16542 runner.gate().await.unwrap();
16543 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16544 assert_ne!(before, after);
16545 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16546 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16547 assert!(!runner.state.gate_allows_progress());
16548 runner.merge().await.unwrap();
16549 assert!(runner.state.merge.is_none());
16550 assert!(
16551 runner
16552 .state
16553 .verification_history
16554 .iter()
16555 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16556 );
16557
16558 runner.state.reviews[0].clean = false;
16560 runner.state.reviews[0].answered = 0;
16561 runner.state.config.verify.e2e = vec!["true".into()];
16562 runner.review_loop().await.unwrap();
16563 runner.gate().await.unwrap();
16564 runner.merge().await.unwrap();
16565 assert_eq!(runner.state.status, RunStatus::Blocked);
16566 assert!(runner.state.merge.is_none());
16567 assert!(
16568 runner
16569 .state
16570 .verification_blocking_cause()
16571 .unwrap()
16572 .contains("review blocked")
16573 );
16574 }
16575
16576 #[cfg(unix)]
16577 #[tokio::test]
16578 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16579 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16580 runner.state.gate_head = None;
16581 runner.state.config.verify.gate =
16582 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16583 runner.review_loop().await.unwrap();
16584 runner.gate().await.unwrap();
16585 assert_ne!(
16586 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16587 old
16588 );
16589 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16590 assert!(!runner.state.gate_allows_progress());
16591 assert_eq!(runner.state.status, RunStatus::Blocked);
16592 runner.merge().await.unwrap();
16593 assert!(runner.state.merge.is_none());
16594 }
16595
16596 #[cfg(unix)]
16597 #[tokio::test]
16598 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16599 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16600 let mut old_review = review_round(true, 0, 1, 1, false, true);
16601 old_review.head = head.clone();
16602 old_review.verified_head = Some(head.clone());
16603 runner.state.reviews = vec![old_review];
16604 runner.state.gate = vec![CommandOutcome {
16605 command: "old gate".into(),
16606 code: None,
16607 output_tail: "timed out after 2400s".into(),
16608 duration_ms: 2_400_000,
16609 resource_blocked: false,
16610 }];
16611 runner.state.gate_ran = true;
16612 runner.state.gate_head = Some(head.clone());
16613 runner.state.land_approval = Some(crate::run::LandApproval {
16614 question: "old-head-approval".into(),
16615 head: head.clone(),
16616 });
16617 runner.state.authorize_base_sync(&task).unwrap();
16618 runner.sync_to_base().await.unwrap();
16619 assert!(!runner.state.base_sync_blocked());
16620 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16621 assert_ne!(head, fresh);
16622 assert!(runner.state.reviews.is_empty());
16623 assert!(!runner.state.gate_ran);
16624 let saved = &runner.state.base_sync_recovery.authorizations[0];
16625 assert_eq!(saved.reviews[0].head, head);
16626 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16627 assert!(saved.gate_ran);
16628 assert!(!saved.gate[0].ok());
16629 runner.review_loop().await.unwrap();
16630 runner.gate().await.unwrap();
16631 assert_eq!(runner.state.reviews[0].head, fresh);
16632 assert_eq!(
16633 runner.state.reviews[0].verified_head.as_deref(),
16634 Some(fresh.as_str())
16635 );
16636 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16637 assert!(runner.state.gate_ran);
16638 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16639 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16640 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16641 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16642 assert_eq!(
16643 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16644 "retained change"
16645 );
16646 }
16647
16648 #[cfg(unix)]
16649 #[tokio::test]
16650 async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16651 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16652 let queue = crate::queue::Queue::open();
16653 let mut owner = queue.get(&task).unwrap();
16654 owner.release_operator("test-resume", None);
16655 queue.put(&mut owner).unwrap();
16656 let generation = owner.resumption.as_ref().unwrap().generation;
16657 runner.state.task_instruction = Some(owner.instruction.clone());
16658 runner.state.driver_exited = true;
16659 runner.state.save().unwrap();
16660 let home = crate::run::home();
16661 let _claim = queue.claim(&task).unwrap();
16662 owner.resumption.as_mut().unwrap().at =
16663 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16664 queue.put(&mut owner).unwrap();
16665 assert!(
16666 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16667 .await
16668 .is_err(),
16669 "an approval from before the stop grants nothing"
16670 );
16671 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16672 queue.put(&mut owner).unwrap();
16673 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16674 .await
16675 .unwrap();
16676 assert!(
16677 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16678 .await
16679 .is_err()
16680 );
16681 runner.state = RunState::load(&runner.state.id).unwrap();
16682 assert_eq!(
16683 runner.state.base_sync_recovery.authorizations[0].generation,
16684 Some(generation)
16685 );
16686 runner.sync_to_base().await.unwrap();
16687 let repo = runner.state.repo.clone();
16688 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16689 git::git(&repo, &["checkout", "main"]).await.unwrap();
16690 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16691 git::git(&repo, &["add", "."]).await.unwrap();
16692 git::git(&repo, &["commit", "-m", "move base again"])
16693 .await
16694 .unwrap();
16695 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16696 git::git(&repo, &["checkout", "fix/retained"])
16697 .await
16698 .unwrap();
16699 runner.finish_after_tally().await.unwrap();
16700 assert_eq!(runner.state.status, RunStatus::Blocked);
16701 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16702 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16703 assert!(!runner.state.base_recovery_pending());
16704 assert!(!crate::handover::can_resume(&runner.state));
16705 assert!(!runner.state.gate_ran);
16706 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16708 queue.put(&mut owner).unwrap();
16709 assert!(
16710 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16711 .await
16712 .is_err()
16713 );
16714 runner.state = RunState::load(&runner.state.id).unwrap();
16715 runner.sync_to_base().await.unwrap();
16716 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16717 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16718 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16719 }
16720
16721 #[cfg(unix)]
16722 #[tokio::test]
16723 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16724 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16725 runner.state.base_sync = None;
16726 runner.state.status = RunStatus::Blocked;
16727 let mut second = runner.state.candidates[0].clone();
16728 second.label = 'B';
16729 runner.state.candidates.push(second);
16730 assert!(runner.state.votes.is_empty());
16731 assert!(runner.state.deliberation.is_empty());
16732 runner.deliberate().await.unwrap();
16733 runner.vote().await.unwrap();
16734 assert_eq!(runner.state.status, RunStatus::Blocked);
16735 assert!(runner.state.votes.is_empty());
16736 assert!(runner.state.deliberation.is_empty());
16737 }
16738
16739 fn runner_at(status: RunStatus) -> Runner {
16742 let mut state = RunState::new(
16743 PathBuf::from("/nonexistent/repo"),
16744 "main".to_owned(),
16745 "deadbeef".to_owned(),
16746 "task".to_owned(),
16747 Config::default(),
16748 );
16749 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16752 state.id = format!(
16753 "20000101-000000-{:04x}",
16754 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16755 );
16756 state.status = status;
16757 Runner {
16758 state,
16759 roles: ResolvedRoles {
16760 implementers: Vec::new(),
16761 judges: Vec::new(),
16762 reviewers: Vec::new(),
16763 fixer: None,
16764 conductor: conductor(),
16765 implementer_roster: Vec::new(),
16766 judge_roster: Vec::new(),
16767 reviewer_roster: Vec::new(),
16768 },
16769 sem: Arc::new(Semaphore::new(1)),
16770 pause: Pause::new(),
16771 interrupt: Pause::new(),
16772 }
16773 }
16774
16775 #[tokio::test]
16776 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16777 crate::run::pin_test_home();
16778 let mut runner = runner_at(RunStatus::Gating);
16779 runner.state.save().unwrap();
16780 let id = runner.state.id.clone();
16781 let passes = runner.state.passes.len();
16782 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16783 let mut claiming = Box::pin(runner.claim_driver());
16784 std::future::poll_fn(|cx| {
16785 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16786 std::task::Poll::Ready(())
16787 })
16788 .await;
16789 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16790 drop(owner);
16791 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16792 .await
16793 .unwrap()
16794 .unwrap();
16795 assert_eq!(execution.runner.state.passes.len(), passes + 1);
16796 assert!(execution.runner.state.resource_block.is_none());
16797 assert_eq!(
16798 execution.runner.state.liveness(false),
16799 crate::run::Liveness::Live
16800 );
16801 drop(execution);
16802 assert!(RunState::load(&id).unwrap().driver_exited);
16803 }
16804
16805 #[tokio::test]
16806 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16807 crate::run::pin_test_home();
16808 let mut runner = runner_at(RunStatus::Gating);
16809 runner.state.save().unwrap();
16810 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16813 assert!(runner.claim_driver().await.is_err());
16814 assert!(runner.state.driver_exited);
16815 assert_eq!(runner.state.driver_pid, None);
16816 let disk = RunState::load(&runner.state.id).unwrap();
16817 assert!(disk.driver_exited);
16818 assert_eq!(disk.driver_pid, None);
16819 assert_eq!(disk.status, RunStatus::Gating);
16820 }
16821
16822 #[tokio::test]
16823 async fn an_early_graph_error_releases_its_driver() {
16824 crate::run::pin_test_home();
16825 let mut runner = runner_at(RunStatus::Blocked);
16826 runner.state.config.graph.selected_implementer = Some("selected".into());
16827 runner.state.save().unwrap();
16828 assert!(runner.execute().await.is_err());
16829 let disk = RunState::load(&runner.state.id).unwrap();
16830 assert!(disk.driver_exited);
16831 assert_eq!(disk.driver_pid, None);
16832 assert_eq!(disk.status, RunStatus::Blocked);
16833 }
16834
16835 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16836 crate::run::MergeRepair {
16837 question: "repair-question".into(),
16838 head: "old-head".into(),
16839 pr: "https://github.com/o/r/pull/42".into(),
16840 reviews: vec![review_round(false, 1, 1, 1, false, true)],
16841 gate: vec![outcome(Some(0), false)],
16842 gate_head: Some("old-head".into()),
16843 gate_fixes: Vec::new(),
16844 pre_gate: Vec::new(),
16845 phase,
16846 failure: None,
16847 }
16848 }
16849
16850 #[tokio::test]
16851 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16852 crate::run::pin_test_home();
16853 let mut runner = runner_at(RunStatus::Landing);
16854 runner.state.merge_repairs.push(merge_repair_request(
16855 crate::run::MergeRepairPhase::Requested,
16856 ));
16857 runner.state.save().unwrap();
16858 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16859 assert!(
16860 runner
16861 .execute()
16862 .await
16863 .unwrap_err()
16864 .to_string()
16865 .contains("already running")
16866 );
16867 assert_eq!(
16868 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16869 crate::run::MergeRepairPhase::Requested
16870 );
16871 drop(claim);
16872 let mut disk = runner.state.clone();
16873 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16874 disk.save().unwrap();
16875 assert!(
16876 runner
16877 .execute()
16878 .await
16879 .unwrap_err()
16880 .to_string()
16881 .contains("decision changed")
16882 );
16883 assert_eq!(
16884 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16885 crate::run::MergeRepairPhase::Complete
16886 );
16887 }
16888
16889 #[test]
16890 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16891 crate::run::pin_test_home();
16892 let mut runner = runner_at(RunStatus::Landing);
16893 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16894 runner.state.reviews = request.reviews.clone();
16895 runner.state.gate = request.gate.clone();
16896 runner.state.gate_ran = true;
16897 runner.state.land_approval = Some(crate::run::LandApproval {
16898 question: "old".into(),
16899 head: "old-head".into(),
16900 });
16901 runner.state.merge_repairs.push(request);
16902 runner.begin_merge_repair_review(0).unwrap();
16903 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16904 assert!(!runner.state.gate_status().ok());
16905 assert!(runner.state.land_approval.is_none());
16906 let disk = RunState::load(&runner.state.id).unwrap();
16907 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16908 assert!(disk.merge_repairs[0].gate[0].ok());
16909 assert_eq!(
16910 disk.merge_repairs[0].phase,
16911 crate::run::MergeRepairPhase::Reviewing
16912 );
16913 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16914 assert!(disk.reviews.is_empty());
16915 }
16916
16917 #[tokio::test]
16918 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16919 crate::run::pin_test_home();
16920 for phase in [
16921 crate::run::MergeRepairPhase::Fixing,
16922 crate::run::MergeRepairPhase::Requested,
16923 ] {
16924 let mut runner = runner_at(RunStatus::Landing);
16925 runner.state.config.graph.land_rounds = 0;
16926 runner.state.merge_repairs.push(merge_repair_request(phase));
16927 runner.state.save().unwrap();
16928 assert!(!runner.repair_merge_review().await.unwrap());
16929 assert_eq!(runner.state.status, RunStatus::Blocked);
16930 assert_eq!(
16931 runner.state.merge_repairs[0].phase,
16932 crate::run::MergeRepairPhase::Failed
16933 );
16934 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16935 assert!(
16936 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16937 "interrupted"
16938 } else {
16939 "budget exhausted"
16940 })
16941 );
16942 assert!(runner.state.merge.is_none());
16943 }
16944 }
16945
16946 #[test]
16950 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16951 crate::run::pin_test_home();
16952 let mut runner = runner_at(RunStatus::Implementing);
16953 let interrupt = Pause::new();
16954 runner.watch_interrupt(interrupt.clone());
16955
16956 interrupt.park_because("task a1b2 asked to run first");
16957
16958 assert!(runner.park_here().expect("park_here"));
16959 assert!(runner.state.parked);
16960 let last = runner.state.events.last().expect("a park event");
16961 assert_eq!(last.node, "park");
16962 assert!(
16963 last.message.contains("task a1b2 asked to run first"),
16964 "expected the interrupt reason in {:?}",
16965 last.message
16966 );
16967 }
16968
16969 #[test]
16972 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16973 crate::run::pin_test_home();
16974 let mut runner = runner_at(RunStatus::Implementing);
16975 runner.state.begin_pass();
16976 let shutdown = Pause::new();
16977 runner.on_pause(shutdown.clone());
16978 let upgrade = ParkCause::Upgrade {
16979 from: "0.121.0".to_owned(),
16980 to: Some("0.121.1".to_owned()),
16981 };
16982 shutdown.park_for(upgrade.clone());
16983 shutdown.park_for(ParkCause::Operator);
16984
16985 assert!(runner.park_here().expect("park_here"));
16986 assert_eq!(
16987 runner.state.passes[0].end,
16988 Some(crate::run::PassEnd::Parked { cause: upgrade })
16989 );
16990 let last = runner.state.events.last().expect("a park event");
16991 assert!(last.message.contains("0.121.0"), "{}", last.message);
16992 }
16993
16994 #[tokio::test]
16995 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16996 crate::run::pin_test_home();
16997 let mut runner = runner_at(RunStatus::Reviewing);
16998 runner.state.save().unwrap();
16999 let mut execution = runner.claim_driver().await.expect("first execution");
17000 assert_eq!(execution.runner.state.passes.len(), 1);
17001 execution
17002 .driver
17003 .finish(&mut execution.runner.state)
17004 .await
17005 .unwrap();
17006 }
17007
17008 #[tokio::test]
17009 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
17010 crate::run::pin_test_home();
17011 let mut runner = runner_at(RunStatus::Reviewing);
17012 runner.state.config.agents = vec![conductor()];
17013 runner.state.config.graph.timeout_resource_wait = 0;
17014 runner.state.save().unwrap();
17015 let id = runner.state.id.clone();
17016 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
17017 assert!(runner.claim_driver().await.is_err());
17018 drop(owner);
17019 drop(runner);
17020 let mut resumed = Runner::resume(&id).unwrap();
17021 assert_eq!(resumed.state.driver_claimed, Some(false));
17022 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
17023 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
17024 let mut execution = resumed
17025 .claim_driver()
17026 .await
17027 .expect("retry initial claim after reload");
17028 assert_eq!(execution.runner.state.driver_claimed, Some(true));
17029 assert_eq!(execution.runner.state.passes.len(), 1);
17030 execution
17031 .driver
17032 .finish(&mut execution.runner.state)
17033 .await
17034 .unwrap();
17035 }
17036
17037 #[tokio::test]
17038 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
17039 crate::run::pin_test_home();
17040 let mut runner = runner_at(RunStatus::Reviewing);
17041 runner.state.config.agents = vec![conductor()];
17042 runner.state.save().unwrap();
17043 let id = runner.state.id.clone();
17044 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
17045 let mut resumed = Runner::resume(&id).unwrap();
17046 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
17047 assert!(resumed.claim_driver().await.is_err());
17048 assert!(RunState::load(&id).unwrap().passes.is_empty());
17049 drop(lease);
17050 let mut execution = resumed.claim_driver().await.unwrap();
17051 execution
17052 .driver
17053 .finish(&mut execution.runner.state)
17054 .await
17055 .unwrap();
17056 }
17057
17058 #[tokio::test]
17059 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
17060 crate::run::pin_test_home();
17061 let mut runner = runner_at(RunStatus::Reviewing);
17062 let state = &mut runner.state;
17063 state.config.agents = vec![conductor()];
17064 state.driver_claimed = None;
17065 state.driver_pid = Some(std::process::id());
17066 state.driver_started_at = None;
17067 state.begin_pass();
17068 state.save().unwrap();
17069 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
17070 std::fs::write(
17071 state.dir().join("driver.json"),
17072 serde_json::to_vec(&serde_json::json!({
17073 "pid": state.driver_pid, "started_at": null,
17074 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
17075 }))
17076 .unwrap(),
17077 )
17078 .unwrap();
17079 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
17080 drop(lease);
17081 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
17082 let mut resumed = Runner::resume(&state.id).unwrap();
17083 assert!(!resumed.state.driver_exited);
17084 let mut execution = resumed
17085 .claim_driver()
17086 .await
17087 .expect("lease proves previous execution ended");
17088 assert_eq!(execution.runner.state.passes.len(), 2);
17089 execution
17090 .driver
17091 .finish(&mut execution.runner.state)
17092 .await
17093 .unwrap();
17094 }
17095
17096 #[tokio::test]
17097 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
17098 crate::run::pin_test_home();
17099 let mut runner = runner_at(RunStatus::Reviewing);
17100 let mut json = serde_json::to_value(&runner.state).unwrap();
17101 json.as_object_mut().unwrap().remove("driver_claimed");
17102 runner.state = serde_json::from_value(json).unwrap();
17103 assert_eq!(runner.state.driver_claimed, None);
17104 runner.state.save().unwrap();
17105 let mut execution = runner
17106 .claim_driver()
17107 .await
17108 .expect("the execution lease, not a missing marker, excludes a second driver");
17109 assert_eq!(execution.runner.state.passes.len(), 1);
17110 execution
17111 .driver
17112 .finish(&mut execution.runner.state)
17113 .await
17114 .unwrap();
17115 }
17116
17117 #[tokio::test]
17118 async fn driver_admission_refuses_a_proven_live_driver() {
17119 crate::run::pin_test_home();
17120 let mut runner = runner_at(RunStatus::Reviewing);
17121 let pid = std::process::id();
17122 runner.state.driver_pid = Some(pid);
17123 runner.state.driver_started_at = crate::proc::process_started_at(pid);
17124 assert!(runner.state.driver_started_at.is_some());
17125 runner.state.save().unwrap();
17126 let error = runner
17127 .claim_driver()
17128 .await
17129 .err()
17130 .expect("a live driver must refuse");
17131 assert!(error.to_string().contains("live driver"), "{error:#}");
17132 let saved = RunState::load(&runner.state.id).unwrap();
17133 assert_eq!(saved.driver_pid, Some(pid));
17134 assert_eq!(saved.status, RunStatus::Reviewing);
17135 assert!(saved.passes.is_empty());
17136 }
17137
17138 #[test]
17139 fn parking_resource_halt_persists_the_pass_once() {
17140 crate::run::pin_test_home();
17141 let mut runner = runner_at(RunStatus::Reviewing);
17142 runner.state.begin_pass();
17143 runner.state.resource_block = Some(crate::run::ResourceBlock {
17144 node: "fix".into(),
17145 kind: "shared build cache".into(),
17146 detail: "cancelled".into(),
17147 at: Timestamp::now(),
17148 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
17149 });
17150 let cause = ParkCause::Upgrade {
17151 from: "old".into(),
17152 to: Some("new".into()),
17153 };
17154 runner.pause.park_for(cause.clone());
17155 runner.stop_for_resource().unwrap();
17158 assert!(runner.resource_halt().unwrap());
17159 assert!(runner.park_here().unwrap());
17160 let saved = RunState::load(&runner.state.id).unwrap();
17161 assert!(saved.parked);
17162 assert_eq!(saved.status, RunStatus::Reviewing);
17163 assert!(saved.passes[0].ended_at.is_some());
17164 assert_eq!(
17165 saved.passes[0].end,
17166 Some(crate::run::PassEnd::Parked {
17167 cause: cause.clone()
17168 })
17169 );
17170 assert_eq!(
17171 saved.resource_block.unwrap().cause,
17172 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
17173 );
17174 assert_eq!(
17175 saved
17176 .events
17177 .iter()
17178 .filter(|event| event.node == "park")
17179 .count(),
17180 1
17181 );
17182 }
17183
17184 #[test]
17192 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
17193 crate::run::pin_test_home();
17194 let mut runner = runner_at(RunStatus::Implementing);
17195 let shutdown = Pause::new();
17196 runner.on_pause(shutdown.clone());
17197 let interrupt = Pause::new();
17198 runner.watch_interrupt(interrupt.clone());
17199
17200 assert!(!runner.park_here().expect("park_here"));
17202 assert!(!runner.state.parked);
17203
17204 interrupt.park_because("test");
17206 assert!(!shutdown.parked());
17207 assert!(runner.park_here().expect("park_here"));
17208 }
17209
17210 #[tokio::test]
17224 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
17225 crate::run::pin_test_home();
17226 let mut runner = runner_at(RunStatus::Implementing);
17227 let interrupt = Pause::new();
17228 runner.watch_interrupt(interrupt.clone());
17229
17230 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
17231 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
17232
17233 let node = async move {
17237 started_tx.send(()).expect("send started");
17238 finish_rx.await.expect("recv finish");
17239 "node finished"
17240 };
17241
17242 let interrupter = async move {
17243 started_rx.await.expect("recv started");
17244 interrupt.park_because("higher-priority task waiting");
17246 tokio::task::yield_now().await;
17250 finish_tx.send(()).expect("send finish");
17251 };
17252
17253 let (node_result, ()) = tokio::join!(node, interrupter);
17254 assert_eq!(
17255 node_result, "node finished",
17256 "the in-flight call ran to completion"
17257 );
17258
17259 assert!(runner.park_here().expect("park_here"));
17262 assert!(runner.state.parked);
17263 }
17264
17265 #[test]
17271 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
17272 crate::run::pin_test_home();
17273 let mut runner = runner_at(RunStatus::Judging);
17274 runner.state.config.agents = vec![conductor()];
17278 runner.state.candidates = vec![Candidate {
17279 index: 0,
17280 label: 'A',
17281 agent: "alpha".to_owned(),
17282 branch: "magi/x/A".to_owned(),
17283 worktree: PathBuf::from("/nonexistent/worktree"),
17284 summary: "did the thing".to_owned(),
17285 stat: "1 file changed".to_owned(),
17286 files: 1,
17287 commits: 1,
17288 empty: false,
17289 failed: None,
17290 verified_noop: None,
17291 duration_ms: 1234,
17292 folded: false,
17293 }];
17294 let run_id = runner.state.id.clone();
17295
17296 let interrupt = Pause::new();
17297 runner.watch_interrupt(interrupt.clone());
17298 interrupt.park_because("task c3d4 asked to run first");
17299 assert!(runner.park_here().expect("park_here"));
17300
17301 let resumed = Runner::resume(&run_id).expect("resume");
17302 assert_eq!(resumed.state.candidates.len(), 1);
17303 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
17304 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
17305 assert_eq!(resumed.state.status, runner.state.status);
17306 assert!(
17307 resumed.state.parked,
17308 "still parked until `execute` actually walks the graph again"
17309 );
17310 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
17311 }
17312
17313 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
17315 let mut q = ask::Question::new(
17316 run.to_owned(),
17317 "implement".to_owned(),
17318 "impl-A".to_owned(),
17319 "Which storage backend should the cache use?".to_owned(),
17320 String::new(),
17321 vec!["SQLite".to_owned(), "Redis".to_owned()],
17322 );
17323 store.put(&mut q).unwrap();
17324 q
17325 }
17326
17327 #[test]
17328 fn a_failed_runs_open_question_is_abandoned() {
17329 ask_test_home();
17330 let store = ask::Questions::open();
17331 let mut runner = runner_at(RunStatus::Failed);
17332 let run = runner.state.id.clone();
17333 let q = ask_open_question(&store, &run);
17334
17335 runner.settle_questions();
17336
17337 let back = store.get(&q.id).unwrap();
17338 assert!(
17339 !back.status.open(),
17340 "the seat that asked died with the run; nobody is left to read an answer"
17341 );
17342 assert!(
17343 back.detail.contains(&run) && back.detail.contains("failed"),
17344 "the reason names what the run became, not just that it is gone: {}",
17345 back.detail
17346 );
17347 }
17348
17349 #[test]
17350 fn a_merged_runs_open_question_is_abandoned_too() {
17351 ask_test_home();
17352 let store = ask::Questions::open();
17353 for status in [RunStatus::Merged, RunStatus::Ready] {
17356 let mut runner = runner_at(status);
17357 let run = runner.state.id.clone();
17358 let q = ask_open_question(&store, &run);
17359
17360 runner.settle_questions();
17361
17362 let back = store.get(&q.id).unwrap();
17363 assert!(
17364 !back.status.open(),
17365 "{status:?} run's question must not outlive the run"
17366 );
17367 }
17368 }
17369
17370 #[test]
17371 fn a_still_resumable_runs_open_question_is_left_alone() {
17372 ask_test_home();
17373 let store = ask::Questions::open();
17374 for status in [RunStatus::Blocked, RunStatus::Stalled] {
17380 let mut runner = runner_at(status);
17381 let run = runner.state.id.clone();
17382 let q = ask_open_question(&store, &run);
17383
17384 runner.settle_questions();
17385
17386 let back = store.get(&q.id).unwrap();
17387 assert!(
17388 back.status.open(),
17389 "{status:?} is still alive; the question must still be waiting"
17390 );
17391 }
17392 }
17393
17394 #[test]
17395 fn settle_questions_never_touches_an_already_answered_question() {
17396 ask_test_home();
17397 let store = ask::Questions::open();
17398 let mut runner = runner_at(RunStatus::Failed);
17399 let run = runner.state.id.clone();
17400 let mut q = ask_open_question(&store, &run);
17401 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
17402 .unwrap();
17403 store.put(&mut q).unwrap();
17404
17405 runner.settle_questions();
17410 runner.settle_questions();
17411
17412 let back = store.get(&q.id).unwrap();
17413 assert_eq!(
17414 back.status,
17415 ask::QuestionStatus::Answered,
17416 "a real answer is a decision on record, never overwritten by a sweep"
17417 );
17418 }
17419
17420 #[tokio::test]
17431 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
17432 crate::run::pin_test_home();
17433 let tmp = tempfile::tempdir().expect("tempdir");
17434 let repo = tmp.path().join("repo");
17435 std::fs::create_dir_all(&repo).unwrap();
17436 init_repo(&repo);
17437
17438 let mut config = Config::default();
17439 config.graph.worktree_root = Some(tmp.path().join("wt"));
17440
17441 let mut state = RunState::new(
17442 repo.clone(),
17443 "main".to_owned(),
17444 "deadbeef".to_owned(),
17445 "task".to_owned(),
17446 config,
17447 );
17448 let root = state.worktree_root();
17449 let wt_a = root.join("cand-A");
17450 let wt_b = root.join("cand-B");
17451 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
17452 .await
17453 .expect("worktree A");
17454 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
17455 .await
17456 .expect("worktree B");
17457
17458 state.candidates = vec![
17459 Candidate {
17460 index: 0,
17461 label: 'A',
17462 agent: "alpha".to_owned(),
17463 branch: "magi/x/A".to_owned(),
17464 worktree: wt_a.clone(),
17465 summary: String::new(),
17466 stat: String::new(),
17467 files: 0,
17468 commits: 0,
17469 empty: false,
17470 failed: None,
17471 verified_noop: None,
17472 duration_ms: 0,
17473 folded: false,
17474 },
17475 Candidate {
17476 index: 1,
17477 label: 'B',
17478 agent: "beta".to_owned(),
17479 branch: "magi/x/B".to_owned(),
17480 worktree: wt_b.clone(),
17481 summary: String::new(),
17482 stat: String::new(),
17483 files: 0,
17484 commits: 0,
17485 empty: false,
17486 failed: None,
17487 verified_noop: None,
17488 duration_ms: 0,
17489 folded: false,
17490 },
17491 ];
17492 state.tally = Some(Tally {
17493 first_choice: BTreeMap::from([('A', 1)]),
17494 borda: BTreeMap::new(),
17495 winner: 'A',
17496 rankings: 1,
17497 unanimous_initial: true,
17498 deliberated: false,
17499 changed_votes: 0,
17500 unanimous_final: true,
17501 tie_break: None,
17502 judges: 1,
17503 present: 1,
17504 quorum: 1,
17505 met_quorum: true,
17506 uncontested: None,
17507 });
17508 state.status = RunStatus::Ready;
17509
17510 fold_run(&mut state, false, &crate::run::home())
17511 .await
17512 .expect("fold_run");
17513
17514 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17515 assert!(
17516 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17517 "the unmerged winner's branch survives"
17518 );
17519 assert!(
17520 !state.candidates[0].folded,
17521 "the winner is not marked folded"
17522 );
17523
17524 assert!(!wt_b.exists(), "the loser's worktree is removed");
17525 assert!(
17526 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17527 "the loser's branch is removed"
17528 );
17529 assert!(state.candidates[1].folded, "the loser is marked folded");
17530 }
17531
17532 #[tokio::test]
17533 #[cfg(unix)]
17534 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17535 use std::os::fd::AsRawFd as _;
17536 use std::time::Duration;
17537
17538 let tmp = tempfile::tempdir().unwrap();
17539 let repo = tmp.path().join("repo");
17540 std::fs::create_dir_all(&repo).unwrap();
17541 init_repo(&repo);
17542 let common = repo.join(".git");
17543 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17544 let home = tmp.path().join("home");
17545 let mut config = Config::default();
17546 config.coordination = crate::config::test_coordination();
17547 config.graph.worktree_root = Some(tmp.path().join("wt"));
17548 config.graph.timeout_resource_wait = 1;
17549 let mut state = RunState::new(
17550 repo.clone(),
17551 "main".into(),
17552 "deadbeef".into(),
17553 "task".into(),
17554 config,
17555 );
17556 let root = state.worktree_root();
17557 let candidate = root.join("cand-A");
17558 let reviewer = root.join("review-1");
17559 let branch = state.branch_for('A');
17560 git::worktree_add_branch(&repo, &candidate, &branch, "main")
17561 .await
17562 .unwrap();
17563 git::worktree_add_detached(&repo, &reviewer, "main")
17564 .await
17565 .unwrap();
17566 git::acquire_worktree_config(&repo).await.unwrap();
17567 state.enabled_worktree_config = true;
17568 state.status = RunStatus::Ready;
17569 state.candidates.push(Candidate {
17570 index: 0,
17571 label: 'A',
17572 agent: "implement".into(),
17573 branch: branch.clone(),
17574 worktree: candidate.clone(),
17575 summary: String::new(),
17576 stat: String::new(),
17577 files: 0,
17578 commits: 0,
17579 empty: false,
17580 failed: None,
17581 verified_noop: None,
17582 duration_ms: 0,
17583 folded: false,
17584 });
17585 state.save_under(&home).unwrap();
17586 let config_before = std::fs::read(common.join("config")).unwrap();
17587 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17588 .await
17589 .unwrap();
17590 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17593 assert_eq!(
17594 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17595 0
17596 );
17597 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17598 assert!(
17599 tokio::time::timeout(Duration::from_millis(200), &mut fold)
17600 .await
17601 .is_err()
17602 );
17603 assert!(candidate.exists());
17604 assert!(reviewer.exists());
17605 assert!(git::branch_exists(&repo, &branch).await.unwrap());
17606 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17607 assert_eq!(
17608 git::git(&repo, &["worktree", "list", "--porcelain"])
17609 .await
17610 .unwrap(),
17611 worktrees_before
17612 );
17613 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17614 .await
17615 .unwrap()
17616 .unwrap_err();
17617 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17618 drop(fold);
17619 assert!(candidate.exists() && reviewer.exists());
17620 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17621 assert_eq!(
17622 state.resource_block.as_ref().unwrap().kind,
17623 "repository coordination"
17624 );
17625 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17626 drop(external);
17627 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17628 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17629 .await
17630 .unwrap()
17631 .unwrap();
17632 drop(fold);
17633 assert!(removed.contains(&branch));
17634 assert!(!root.exists());
17635 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17636 assert!(state.candidates[0].folded);
17637 assert!(!state.enabled_worktree_config);
17638 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17639 let enabled = git::git_raw(
17640 &repo,
17641 &["config", "--local", "--get", "extensions.worktreeConfig"],
17642 )
17643 .await
17644 .unwrap();
17645 assert!(!enabled.ok(), "the last config reference was released");
17646 }
17647
17648 #[tokio::test]
17651 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17652 let tmp = tempfile::tempdir().expect("tempdir");
17653 let repo = tmp.path().join("repo");
17654 std::fs::create_dir_all(&repo).unwrap();
17655 init_repo(&repo);
17656 let home = tmp.path().join("home");
17657
17658 let mut config = Config::default();
17659 config.graph.worktree_root = Some(tmp.path().join("wt"));
17660 let mut state = RunState::new(
17661 repo.clone(),
17662 "main".to_owned(),
17663 "deadbeef".to_owned(),
17664 "task".to_owned(),
17665 config,
17666 );
17667 git::git(&repo, &["branch", "magi/x/A", "main"])
17669 .await
17670 .expect("branch");
17671 state.candidates = vec![Candidate {
17672 index: 0,
17673 label: 'A',
17674 agent: "alpha".to_owned(),
17675 branch: "magi/x/A".to_owned(),
17676 worktree: state.worktree_root().join("cand-A"),
17677 summary: String::new(),
17678 stat: String::new(),
17679 files: 0,
17680 commits: 0,
17681 empty: false,
17682 failed: None,
17683 verified_noop: None,
17684 duration_ms: 0,
17685 folded: true,
17686 }];
17687 state.released_to = Some("20260901-000000-new1".to_owned());
17688 state.released_branches = vec!["magi/x/A".to_owned()];
17689
17690 fold_run(&mut state, true, &home).await.expect("fold_run");
17691 state.status = RunStatus::Merged;
17692 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17693 state.merge = Some(crate::run::MergeOutcome {
17694 mode: MergeMode::Pr,
17695 ok: true,
17696 detail: "confirmed merged".into(),
17697 empty: false,
17698 });
17699 fold_run(&mut state, true, &home)
17700 .await
17701 .expect("merged fold");
17702
17703 assert!(
17704 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17705 "the handed-over branch survives a fold"
17706 );
17707 }
17708
17709 #[tokio::test]
17713 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17714 let tmp = tempfile::tempdir().expect("tempdir");
17715 let repo = tmp.path().join("repo");
17716 std::fs::create_dir_all(&repo).unwrap();
17717 init_repo(&repo);
17718 let run = |args: &[&str]| {
17719 let out = std::process::Command::new("git")
17720 .quiet()
17721 .args(args)
17722 .current_dir(&repo)
17723 .output()
17724 .expect("spawn git");
17725 assert!(out.status.success(), "git {args:?}");
17726 };
17727 run(&["branch", "magi/x/A"]);
17728 run(&["checkout", "-q", "-b", "magi/x/B"]);
17729 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17730 run(&["add", "-A"]);
17731 run(&["commit", "-q", "-m", "work"]);
17732 run(&["checkout", "-q", "main"]);
17733
17734 let probe = RunState::new(
17738 repo.clone(),
17739 "main".to_owned(),
17740 "deadbeef".to_owned(),
17741 "task".to_owned(),
17742 Config::default(),
17743 );
17744 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17745 let bare = tmp.path().join("origin.git");
17746 let out = std::process::Command::new("git")
17747 .quiet()
17748 .args(["init", "-q", "--bare"])
17749 .arg(&bare)
17750 .output()
17751 .expect("spawn git");
17752 assert!(out.status.success(), "git init --bare");
17753 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17754 run(&["push", "-q", "origin", "main"]);
17755
17756 let mut state = RunState::new(
17757 repo.clone(),
17758 "main".to_owned(),
17759 "deadbeef".to_owned(),
17760 "task".to_owned(),
17761 Config::default(),
17762 );
17763 state.seeds = vec![refs::Seed {
17764 token: "magi/27b2/A".to_owned(),
17765 kind: refs::SeedKind::Unresolved,
17766 sha: String::new(),
17767 branch: true,
17768 detail: "no branch or commit named magi/27b2/A".to_owned(),
17769 }];
17770
17771 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17772 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17773 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17774 let detail = empty_candidate_detail(&state, "main");
17775 assert!(detail.starts_with("empty candidate"), "{detail}");
17776 assert!(!detail.contains("magi/27b2/A"), "{detail}");
17777 assert!(detail.contains("--start-from"), "{detail}");
17778 }
17779
17780 #[tokio::test]
17789 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17790 let tmp = tempfile::tempdir().expect("tempdir");
17791 let repo = tmp.path().join("repo");
17792 std::fs::create_dir_all(&repo).unwrap();
17793 init_repo(&repo);
17794
17795 let mut config = Config::default();
17796 config.merge.mode = MergeMode::Local;
17797
17798 let mut state = RunState::new(
17799 repo.clone(),
17800 "main".to_owned(),
17801 "deadbeef".to_owned(),
17802 "task".to_owned(),
17803 config,
17804 );
17805 state.candidates = vec![Candidate {
17806 index: 0,
17807 label: 'A',
17808 agent: "alpha".to_owned(),
17809 branch: "does-not-exist".to_owned(),
17810 worktree: repo.clone(),
17811 summary: String::new(),
17812 stat: String::new(),
17813 files: 0,
17814 commits: 0,
17815 empty: false,
17816 failed: None,
17817 verified_noop: None,
17818 duration_ms: 0,
17819 folded: false,
17820 }];
17821 state.tally = Some(Tally {
17822 first_choice: BTreeMap::from([('A', 1)]),
17823 borda: BTreeMap::new(),
17824 winner: 'A',
17825 rankings: 1,
17826 unanimous_initial: true,
17827 deliberated: false,
17828 changed_votes: 0,
17829 unanimous_final: true,
17830 tie_break: None,
17831 judges: 0,
17832 present: 0,
17833 quorum: 0,
17834 met_quorum: true,
17835 uncontested: Some("only candidate A produced a change".to_owned()),
17836 });
17837 state.reviews = vec![ReviewRound {
17838 round: 1,
17839 head: "deadbeef".to_owned(),
17840 verified_head: None,
17841 verified_at: None,
17842 reviews: Vec::new(),
17843 e2e: Vec::new(),
17844 fix: None,
17845 blocking: 0,
17846 answered: 0,
17847 expected: 0,
17848 clean: true,
17849 verify_retried: false,
17850 e2e_deferred: false,
17851 e2e_defer_reason: None,
17852 progressed: false,
17853 vote_split: false,
17854 reconsideration: Vec::new(),
17855 verdict: None,
17856 }];
17857 state.gate = vec![CommandOutcome {
17858 command: "test".to_owned(),
17859 code: Some(0),
17860 output_tail: String::new(),
17861 duration_ms: 0,
17862 resource_blocked: false,
17863 }];
17864 state.gate_ran = true;
17865 state.status = RunStatus::Ready;
17870 state.merge = Some(MergeOutcome {
17871 mode: MergeMode::Local,
17872 ok: false,
17873 detail: "already concluded".to_owned(),
17874 empty: false,
17875 });
17876
17877 let mut runner = Runner {
17878 state,
17879 roles: ResolvedRoles {
17880 implementers: Vec::new(),
17881 judges: Vec::new(),
17882 reviewers: Vec::new(),
17883 fixer: None,
17884 conductor: conductor(),
17885 implementer_roster: Vec::new(),
17886 judge_roster: Vec::new(),
17887 reviewer_roster: Vec::new(),
17888 },
17889 sem: Arc::new(Semaphore::new(1)),
17890 pause: Pause::new(),
17891 interrupt: Pause::new(),
17892 };
17893
17894 runner.merge().await.expect("merge");
17895
17896 assert_eq!(
17897 runner.state.status,
17898 RunStatus::Ready,
17899 "a concluded run's status must not change on reentry"
17900 );
17901 assert_eq!(
17902 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17903 Some("already concluded"),
17904 "merge must not run again once the node already recorded an outcome"
17905 );
17906 }
17907
17908 #[tokio::test]
17917 async fn merge_refuses_a_gate_that_has_not_actually_run() {
17918 let tmp = tempfile::tempdir().expect("tempdir");
17919 let repo = tmp.path().join("repo");
17920 std::fs::create_dir_all(&repo).unwrap();
17921 init_repo(&repo);
17922
17923 let mut config = Config::default();
17924 config.merge.mode = MergeMode::Local;
17925
17926 let mut state = RunState::new(
17927 repo.clone(),
17928 "main".to_owned(),
17929 "deadbeef".to_owned(),
17930 "task".to_owned(),
17931 config,
17932 );
17933 state.candidates = vec![Candidate {
17934 index: 0,
17935 label: 'A',
17936 agent: "alpha".to_owned(),
17937 branch: "does-not-exist".to_owned(),
17938 worktree: repo.clone(),
17939 summary: String::new(),
17940 stat: String::new(),
17941 files: 0,
17942 commits: 0,
17943 empty: false,
17944 failed: None,
17945 verified_noop: None,
17946 duration_ms: 0,
17947 folded: false,
17948 }];
17949 state.tally = Some(Tally {
17950 first_choice: BTreeMap::from([('A', 1)]),
17951 borda: BTreeMap::new(),
17952 winner: 'A',
17953 rankings: 1,
17954 unanimous_initial: true,
17955 deliberated: false,
17956 changed_votes: 0,
17957 unanimous_final: true,
17958 tie_break: None,
17959 judges: 0,
17960 present: 0,
17961 quorum: 0,
17962 met_quorum: true,
17963 uncontested: Some("only candidate A produced a change".to_owned()),
17964 });
17965 state.reviews = vec![ReviewRound {
17966 round: 1,
17967 head: "deadbeef".to_owned(),
17968 verified_head: None,
17969 verified_at: None,
17970 reviews: Vec::new(),
17971 e2e: Vec::new(),
17972 fix: None,
17973 blocking: 0,
17974 answered: 0,
17975 expected: 0,
17976 clean: true,
17977 verify_retried: false,
17978 e2e_deferred: false,
17979 e2e_defer_reason: None,
17980 progressed: false,
17981 vote_split: false,
17982 reconsideration: Vec::new(),
17983 verdict: None,
17984 }];
17985 state.gate = Vec::new();
17987 state.gate_ran = false;
17988 state.status = RunStatus::Gating;
17989
17990 let mut runner = Runner {
17991 state,
17992 roles: ResolvedRoles {
17993 implementers: Vec::new(),
17994 judges: Vec::new(),
17995 reviewers: Vec::new(),
17996 fixer: None,
17997 conductor: conductor(),
17998 implementer_roster: Vec::new(),
17999 judge_roster: Vec::new(),
18000 reviewer_roster: Vec::new(),
18001 },
18002 sem: Arc::new(Semaphore::new(1)),
18003 pause: Pause::new(),
18004 interrupt: Pause::new(),
18005 };
18006
18007 runner.merge().await.expect("merge");
18008
18009 assert!(
18010 runner.state.merge.is_none(),
18011 "an empty gate must never be read as a passing one: {:?}",
18012 runner.state.merge
18013 );
18014 }
18015
18016 #[tokio::test]
18023 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
18024 let tmp = tempfile::tempdir().expect("tempdir");
18025 let repo = tmp.path().join("repo");
18026 std::fs::create_dir_all(&repo).unwrap();
18027 init_repo(&repo);
18028
18029 let config = Config::default();
18031
18032 let mut state = RunState::new(
18033 repo.clone(),
18034 "main".to_owned(),
18035 "deadbeef".to_owned(),
18036 "task".to_owned(),
18037 config,
18038 );
18039 state.candidates = vec![Candidate {
18040 index: 0,
18041 label: 'A',
18042 agent: "alpha".to_owned(),
18043 branch: "does-not-exist".to_owned(),
18044 worktree: repo.clone(),
18045 summary: String::new(),
18046 stat: String::new(),
18047 files: 0,
18048 commits: 0,
18049 empty: false,
18050 failed: None,
18051 verified_noop: None,
18052 duration_ms: 0,
18053 folded: false,
18054 }];
18055 state.tally = Some(Tally {
18056 first_choice: BTreeMap::from([('A', 1)]),
18057 borda: BTreeMap::new(),
18058 winner: 'A',
18059 rankings: 1,
18060 unanimous_initial: true,
18061 deliberated: false,
18062 changed_votes: 0,
18063 unanimous_final: true,
18064 tie_break: None,
18065 judges: 0,
18066 present: 0,
18067 quorum: 0,
18068 met_quorum: true,
18069 uncontested: Some("only candidate A produced a change".to_owned()),
18070 });
18071 state.reviews = vec![ReviewRound {
18072 round: 1,
18073 head: "deadbeef".to_owned(),
18074 verified_head: None,
18075 verified_at: None,
18076 reviews: Vec::new(),
18077 e2e: Vec::new(),
18078 fix: None,
18079 blocking: 0,
18080 answered: 0,
18081 expected: 0,
18082 clean: true,
18083 verify_retried: false,
18084 e2e_deferred: false,
18085 e2e_defer_reason: None,
18086 progressed: false,
18087 vote_split: false,
18088 reconsideration: Vec::new(),
18089 verdict: None,
18090 }];
18091
18092 let mut runner = Runner {
18093 state,
18094 roles: ResolvedRoles {
18095 implementers: Vec::new(),
18096 judges: Vec::new(),
18097 reviewers: Vec::new(),
18098 fixer: None,
18099 conductor: conductor(),
18100 implementer_roster: Vec::new(),
18101 judge_roster: Vec::new(),
18102 reviewer_roster: Vec::new(),
18103 },
18104 sem: Arc::new(Semaphore::new(1)),
18105 pause: Pause::new(),
18106 interrupt: Pause::new(),
18107 };
18108
18109 runner.gate().await.expect("gate");
18110 assert!(
18111 runner.state.gate_ran,
18112 "zero configured commands is still a real attempt, not an unrun gate"
18113 );
18114 assert!(runner.state.gate.is_empty());
18115 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
18116 assert_ne!(
18117 runner.state.status,
18118 RunStatus::Blocked,
18119 "a gate with nothing to check must not read as failed"
18120 );
18121
18122 runner.merge().await.expect("merge");
18123 assert_eq!(
18124 runner.state.status,
18125 RunStatus::Ready,
18126 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
18127 );
18128 }
18129
18130 #[tokio::test]
18141 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
18142 crate::run::pin_test_home();
18143 let home = crate::run::home();
18144
18145 let tmp = tempfile::tempdir().expect("tempdir");
18146 let repo = tmp.path().join("repo");
18147 std::fs::create_dir_all(&repo).unwrap();
18148 init_repo(&repo);
18149 let cache_dir = tmp.path().join("target");
18152
18153 let mut config = Config::default();
18154 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
18155 config.graph.timeout_verify = Some(2);
18159 config.graph.timeout_resource_wait = 0;
18160
18161 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18162 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18163 .expect("no io error acquiring directly")
18164 {
18165 crate::cache::AcquireOutcome::Acquired(g) => g,
18166 crate::cache::AcquireOutcome::Busy(b) => {
18167 panic!("expected the direct acquire to win the lease first: {b:?}")
18168 }
18169 };
18170
18171 let mut state = RunState::new(
18172 repo.clone(),
18173 "main".to_owned(),
18174 "deadbeef".to_owned(),
18175 "task".to_owned(),
18176 config,
18177 );
18178 state.candidates = vec![Candidate {
18179 index: 0,
18180 label: 'A',
18181 agent: "alpha".to_owned(),
18182 branch: "does-not-exist".to_owned(),
18183 worktree: repo.clone(),
18184 summary: String::new(),
18185 stat: String::new(),
18186 files: 0,
18187 commits: 0,
18188 empty: false,
18189 failed: None,
18190 verified_noop: None,
18191 duration_ms: 0,
18192 folded: false,
18193 }];
18194 state.tally = Some(Tally {
18195 first_choice: BTreeMap::from([('A', 1)]),
18196 borda: BTreeMap::new(),
18197 winner: 'A',
18198 rankings: 1,
18199 unanimous_initial: true,
18200 deliberated: false,
18201 changed_votes: 0,
18202 unanimous_final: true,
18203 tie_break: None,
18204 judges: 0,
18205 present: 0,
18206 quorum: 0,
18207 met_quorum: true,
18208 uncontested: Some("only candidate A produced a change".to_owned()),
18209 });
18210 state.reviews = vec![ReviewRound {
18211 round: 1,
18212 head: "deadbeef".to_owned(),
18213 verified_head: None,
18214 verified_at: None,
18215 reviews: Vec::new(),
18216 e2e: Vec::new(),
18217 fix: None,
18218 blocking: 0,
18219 answered: 0,
18220 expected: 0,
18221 clean: true,
18222 verify_retried: false,
18223 e2e_deferred: false,
18224 e2e_defer_reason: None,
18225 progressed: false,
18226 vote_split: false,
18227 reconsideration: Vec::new(),
18228 verdict: None,
18229 }];
18230
18231 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
18232 state.reviews[0].e2e = vec![CommandOutcome {
18233 command: state.config.verify.e2e[0].clone(),
18234 code: Some(0),
18235 output_tail: String::new(),
18236 duration_ms: 0,
18237 resource_blocked: false,
18238 }];
18239 let mut runner = Runner {
18240 state,
18241 roles: ResolvedRoles {
18242 implementers: Vec::new(),
18243 judges: Vec::new(),
18244 reviewers: Vec::new(),
18245 fixer: None,
18246 conductor: conductor(),
18247 implementer_roster: Vec::new(),
18248 judge_roster: Vec::new(),
18249 reviewer_roster: Vec::new(),
18250 },
18251 sem: Arc::new(Semaphore::new(1)),
18252 pause: Pause::new(),
18253 interrupt: Pause::new(),
18254 };
18255
18256 runner.gate().await.expect("gate");
18257 assert!(runner.state.resource_block.is_none());
18258 assert!(
18259 !runner
18260 .state
18261 .events
18262 .iter()
18263 .any(|event| { event.message.contains("waiting for the shared build cache") }),
18264 "a gate with nothing to run must never request the held cache lease"
18265 );
18266 assert!(
18267 runner.state.gate_ran,
18268 "zero commands is still a real, immediate attempt"
18269 );
18270 assert!(runner.state.gate.is_empty());
18271 assert_ne!(
18272 runner.state.status,
18273 RunStatus::Blocked,
18274 "must not read as resource-blocked on a lease it never asked for"
18275 );
18276 }
18277
18278 #[cfg(unix)]
18281 #[tokio::test]
18282 async fn verification_is_standalone_inside_a_live_worker() {
18283 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
18284 if std::env::var_os(CHILD).is_none() {
18285 let home = tempfile::tempdir().unwrap();
18286 let generation = crate::worker::pin_current(home.path()).unwrap();
18287 let output = std::process::Command::new(std::env::current_exe().unwrap())
18288 .args([
18289 "--exact",
18290 "graph::tests::verification_is_standalone_inside_a_live_worker",
18291 "--nocapture",
18292 ])
18293 .env(CHILD, "1")
18294 .env("MAGI_HOME", home.path())
18295 .env("MAGI_WORKER_HOME", home.path())
18296 .env("MAGI_WORKER_ID", "abcdef")
18297 .env("MAGI_WORKER_GENERATION", &generation.id)
18298 .env("MAGI_WORKER_EXE", &generation.executable)
18299 .env("MAGI_RUN", "verification-parent")
18300 .env("MAGI_NODE", "gate")
18301 .env("MAGI_SEAT", "worker-fixture")
18302 .env("MAGI_PROCESS_OWNER", "fixture-owner")
18303 .env("MAGI_NO_AUTOUPDATE", "0")
18304 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
18305 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
18306 .env(
18307 "VERIFY_EXPECT_TOOLCHAIN",
18308 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
18309 )
18310 .env(
18311 "VERIFY_EXPECT_CACHE",
18312 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
18313 )
18314 .output()
18315 .unwrap();
18316 assert!(
18317 output.status.success()
18318 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
18319 "{}\n{}",
18320 String::from_utf8_lossy(&output.stdout),
18321 String::from_utf8_lossy(&output.stderr)
18322 );
18323 return;
18324 }
18325 let home = crate::run::home();
18326 let directory = home.join("workers/jobs/abcdef");
18327 std::fs::create_dir_all(&directory).unwrap();
18328 let job = crate::worker::Job {
18329 id: "abcdef".into(),
18330 sequence: 1,
18331 kind: "task".into(),
18332 key: "task:verification".into(),
18333 payload: serde_json::json!({}),
18334 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
18335 version: Some(env!("CARGO_PKG_VERSION").into()),
18336 status: crate::worker::Status::Running,
18337 result: None,
18338 pid: Some(std::process::id()),
18339 created_at: 0,
18340 finished_at: None,
18341 };
18342 std::fs::write(
18343 directory.join("job.json"),
18344 serde_json::to_vec(&job).unwrap(),
18345 )
18346 .unwrap();
18347 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
18348 let assert_worker = || {
18349 assert_eq!(
18350 crate::worker::current_job(&home).unwrap().unwrap().id,
18351 job.id
18352 );
18353 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
18354 assert_eq!(
18355 crate::worker::helper_executable(&home).unwrap(),
18356 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
18357 );
18358 assert!(
18359 crate::updater::Checker::new(&crate::config::Update {
18360 mode: crate::config::UpdateMode::Notify,
18361 interval: None,
18362 auto_restart: false,
18363 })
18364 .is_none(),
18365 "the driver remains pinned, even while tests are standalone"
18366 );
18367 };
18368 assert_worker();
18369 let repo = home.join("repo");
18370 std::fs::create_dir_all(&repo).unwrap();
18371 init_repo(&repo);
18372 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
18375 test -z "${MAGI_WORKER_GENERATION+x}" &&
18376 test -z "${MAGI_WORKER_HOME+x}" &&
18377 test -z "${MAGI_WORKER_EXE+x}" &&
18378 test "$MAGI_HOME/repo" -ef . &&
18379 test "$MAGI_RUN" = verification-parent &&
18380 test "$MAGI_NODE" = gate &&
18381 test "$MAGI_SEAT" = worker-fixture &&
18382 test -n "$MAGI_PROCESS_OWNER" &&
18383 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
18384 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
18385 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
18386 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
18387 assert!(
18389 !std::process::Command::new("sh")
18390 .args(["-c", probe])
18391 .current_dir(&repo)
18392 .status()
18393 .unwrap()
18394 .success()
18395 );
18396 let unrelated_home = tempfile::tempdir().unwrap();
18397 assert!(
18398 crate::worker::current_job(unrelated_home.path()).is_err(),
18399 "an inherited worker id cannot be resolved in a unit fixture's home"
18400 );
18401 let mut runner = runner_at(RunStatus::Gating);
18402 runner.state.repo = repo.clone();
18403 runner.state.config.disk.min_free_bytes = 0;
18404 runner.state.config.coordination = crate::config::test_coordination();
18405 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
18406 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
18407 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
18408 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
18409 let winner = Candidate {
18410 index: 0,
18411 label: 'A',
18412 agent: "fixture".into(),
18413 branch: "main".into(),
18414 worktree: repo.clone(),
18415 summary: String::new(),
18416 stat: String::new(),
18417 files: 0,
18418 commits: 0,
18419 empty: false,
18420 failed: None,
18421 verified_noop: None,
18422 duration_ms: 0,
18423 folded: false,
18424 };
18425 let (e2e, retried, pids) = run_e2e_with_retry(
18426 &mut runner.state,
18427 &["sh".into(), "-c".into()],
18428 &[format!("{probe} && printf initial")],
18429 &repo,
18430 Duration::from_secs(10),
18431 "initial",
18432 )
18433 .await;
18434 assert!(!retried && pids.is_empty());
18435 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
18436 assert_eq!(e2e[0].output_tail, "initial");
18437 let gate = runner.run_gate(&winner).await.unwrap();
18438 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
18439 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
18442 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
18443 passed: false,
18444 outcomes: Vec::new(),
18445 });
18446 runner.state.save().unwrap();
18447 let mut resumed = RunState::load(&runner.state.id).unwrap();
18448 let wait = crate::repo_lock::WaitPolicy::default();
18449 assert!(matches!(
18450 check_gate_repair_e2e(&mut resumed, &winner, &wait)
18451 .await
18452 .unwrap(),
18453 GateFix::Retry
18454 ));
18455 let saved = RunState::load(&resumed.id).unwrap();
18456 let repair = saved.gate_repair_e2e.unwrap();
18457 assert!(repair.passed);
18458 assert_eq!(repair.outcomes.len(), 1);
18459 assert!(repair.outcomes[0].ok());
18460 let landing = run_landing_gate(&mut resumed, &winner, &wait)
18461 .await
18462 .unwrap();
18463 assert_eq!(landing.len(), 1);
18464 assert!(landing[0].ok(), "{landing:?}");
18465 assert_eq!(
18466 std::fs::read_to_string(repo.join("phases")).unwrap(),
18467 "pre\ngate\ne2e\npre\ngate\n"
18468 );
18469 assert_worker();
18470 }
18471
18472 #[cfg(unix)]
18473 #[tokio::test]
18474 async fn verification_keeps_the_panic_and_the_full_output() {
18475 crate::run::pin_test_home();
18476 let tmp = tempfile::tempdir().unwrap();
18477 let mut config = Config::default();
18478 config.disk.min_free_bytes = 0;
18479 let mut state = RunState::new(
18480 tmp.path().to_owned(),
18481 "main".into(),
18482 "base".into(),
18483 "verification evidence".into(),
18484 config,
18485 );
18486 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
18487 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
18488 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
18489 let mut names = Vec::new();
18490 for _ in 0..2 {
18491 let (outcomes, _) = run_commands(
18492 &mut state,
18493 "gate",
18494 "gate",
18495 0,
18496 &["sh".into(), "-c".into()],
18497 &[script.into()],
18498 tmp.path(),
18499 Duration::from_secs(60),
18500 )
18501 .await;
18502 let tail = &outcomes[0].output_tail;
18503 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18504 assert!(tail.contains("boom assertion"));
18505 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18506 let line = tail.lines().next().unwrap();
18507 names.push(line.to_owned());
18508 }
18509 assert_ne!(names[0], names[1], "the second run must not overwrite");
18510 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18511 assert!(full.len() > 500_000 && full.contains("boom assertion"));
18512 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18513
18514 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18516 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18517 let (outcomes, _) = run_commands(
18518 &mut state,
18519 "gate",
18520 "gate",
18521 0,
18522 &["sh".into(), "-c".into()],
18523 &["echo failure-output; exit 7".into()],
18524 tmp.path(),
18525 Duration::from_secs(10),
18526 )
18527 .await;
18528 assert_eq!(outcomes[0].code, Some(7));
18529 assert!(state.events.iter().any(|e| {
18530 e.node == "gate" && e.message.contains("could not save the full gate output:")
18531 }));
18532 }
18533
18534 #[cfg(unix)]
18535 #[tokio::test]
18536 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18537 crate::run::pin_test_home();
18538 let tmp = tempfile::tempdir().unwrap();
18539 let mut config = Config::default();
18540 config.disk.min_free_bytes = 0;
18541 let mut state = RunState::new(
18542 tmp.path().to_owned(),
18543 "main".into(),
18544 "base".into(),
18545 "verification evidence".into(),
18546 config,
18547 );
18548 let (outcomes, pids) = run_commands(
18549 &mut state,
18550 "gate",
18551 "gate",
18552 1,
18553 &["sh".into(), "-c".into()],
18554 &[
18555 "printf before-timeout; exec sleep 30".into(),
18556 "kill -TERM $$".into(),
18557 "exit 7".into(),
18558 "printf success".into(),
18559 ],
18560 tmp.path(),
18561 Duration::from_secs(1),
18562 )
18563 .await;
18564 wait_for_timed_out_children_to_die(&pids).await;
18565 assert_eq!(
18566 pids.len(),
18567 1,
18568 "unleased timeout retains its death-check PID"
18569 );
18570 assert_eq!(outcomes.len(), 4);
18571 assert_eq!(outcomes[0].code, None);
18572 assert!(outcomes[0].output_tail.contains("before-timeout"));
18573 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18574 assert_eq!(outcomes[1].code, None);
18575 assert!(outcomes[1].output_tail.contains("command terminated:"));
18576 assert!(!outcomes[1].output_tail.contains("timed out"));
18577 assert_eq!(outcomes[2].code, Some(7));
18578 assert!(!outcomes[2].output_tail.contains("timed out"));
18579 assert_eq!(outcomes[3].code, Some(0));
18580 assert_eq!(outcomes[3].output_tail, "success");
18581 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18582 }
18583
18584 #[tokio::test]
18594 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18595 crate::run::pin_test_home();
18596
18597 let tmp = tempfile::tempdir().expect("tempdir");
18598 let repo = tmp.path().join("repo");
18599 std::fs::create_dir_all(&repo).unwrap();
18600 init_repo(&repo);
18601
18602 let mut config = Config::default();
18603 config.verify.gate = vec![
18604 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18605 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18606 .to_owned(),
18607 ];
18608
18609 let mut state = RunState::new(
18610 repo.clone(),
18611 "main".to_owned(),
18612 "deadbeef".to_owned(),
18613 "task".to_owned(),
18614 config,
18615 );
18616 let run_id = state.id.clone();
18617 state.candidates = vec![Candidate {
18618 index: 0,
18619 label: 'A',
18620 agent: "alpha".to_owned(),
18621 branch: "does-not-exist".to_owned(),
18622 worktree: repo.clone(),
18623 summary: String::new(),
18624 stat: String::new(),
18625 files: 0,
18626 commits: 0,
18627 empty: false,
18628 failed: None,
18629 verified_noop: None,
18630 duration_ms: 0,
18631 folded: false,
18632 }];
18633 state.tally = Some(Tally {
18634 first_choice: BTreeMap::from([('A', 1)]),
18635 borda: BTreeMap::new(),
18636 winner: 'A',
18637 rankings: 1,
18638 unanimous_initial: true,
18639 deliberated: false,
18640 changed_votes: 0,
18641 unanimous_final: true,
18642 tie_break: None,
18643 judges: 0,
18644 present: 0,
18645 quorum: 0,
18646 met_quorum: true,
18647 uncontested: Some("only candidate A produced a change".to_owned()),
18648 });
18649 state.reviews = vec![ReviewRound {
18650 round: 1,
18651 head: "deadbeef".to_owned(),
18652 verified_head: None,
18653 verified_at: None,
18654 reviews: Vec::new(),
18655 e2e: Vec::new(),
18656 fix: None,
18657 blocking: 0,
18658 answered: 0,
18659 expected: 0,
18660 clean: true,
18661 verify_retried: false,
18662 e2e_deferred: false,
18663 e2e_defer_reason: None,
18664 progressed: false,
18665 vote_split: false,
18666 reconsideration: Vec::new(),
18667 verdict: None,
18668 }];
18669
18670 let mut runner = Runner {
18671 state,
18672 roles: ResolvedRoles {
18673 implementers: Vec::new(),
18674 judges: Vec::new(),
18675 reviewers: Vec::new(),
18676 fixer: None,
18677 conductor: conductor(),
18678 implementer_roster: Vec::new(),
18679 judge_roster: Vec::new(),
18680 reviewer_roster: Vec::new(),
18681 },
18682 sem: Arc::new(Semaphore::new(1)),
18683 pause: Pause::new(),
18684 interrupt: Pause::new(),
18685 };
18686
18687 let started_marker = repo.join("started.marker");
18688 let release_marker = repo.join("release.marker");
18689 let poller = tokio::spawn(async move {
18690 for _ in 0..100 {
18695 if started_marker.exists()
18696 && let Ok(s) = crate::run::RunState::load(&run_id)
18697 && let Some(a) = s.active.get("gate")
18698 {
18699 std::fs::write(&release_marker, b"go").expect("release marker");
18700 return Some(a.clone());
18701 }
18702 tokio::time::sleep(Duration::from_millis(50)).await;
18703 }
18704 None
18705 });
18706
18707 runner.gate().await.expect("gate");
18708 let captured = poller.await.expect("poller task");
18709 let captured = captured.expect(
18710 "the poller never saw a `gate` task entry in run.json while the command was \
18711 still blocked on its own release marker",
18712 );
18713
18714 assert_eq!(captured.task.as_deref(), Some("gate"));
18715 assert_eq!(captured.node, "gate");
18716 assert_eq!(captured.index, Some(1));
18717 assert_eq!(captured.total, Some(1));
18718 assert!(
18719 captured
18720 .command
18721 .as_deref()
18722 .is_some_and(|c| c.contains("started.marker")),
18723 "{captured:?}"
18724 );
18725
18726 assert!(
18727 runner.state.active.is_empty(),
18728 "the entry must be cleared once the command actually finished: {:?}",
18729 runner.state.active
18730 );
18731 assert!(runner.state.gate_ran);
18732 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18733 }
18734
18735 #[tokio::test]
18738 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18739 use crate::verdict::{Finding, ReviewVote, Severity};
18740 crate::run::pin_test_home();
18741 let tmp = tempfile::tempdir().expect("tempdir");
18742 let repo = tmp.path().join("repo");
18743 std::fs::create_dir_all(&repo).unwrap();
18744 init_repo(&repo);
18745
18746 for (severity, vote, expect) in [
18747 (Severity::Major, ReviewVote::Reject, true),
18748 (Severity::Minor, ReviewVote::Reject, false),
18749 (Severity::Major, ReviewVote::Approve, false),
18750 ] {
18751 let mut round = review_round(false, 1, 1, 1, false, true);
18752 round.reviews = vec![ReviewRecord {
18753 reviewer: 1,
18754 agent: "alpha".to_owned(),
18755 summary: String::new(),
18756 findings: vec![Finding {
18757 id: "R1-1-1".to_owned(),
18758 severity,
18759 file: None,
18760 line: None,
18761 title: "t".to_owned(),
18762 detail: String::new(),
18763 }],
18764 vote: Some(vote),
18765 failed: None,
18766 duration_ms: 0,
18767 attempts: 0,
18768 }];
18769 let mut state = RunState::new(
18770 repo.clone(),
18771 "main".to_owned(),
18772 "deadbeef".to_owned(),
18773 "task".to_owned(),
18774 Config::default(),
18775 );
18776 state.reviews = vec![round];
18777 let mut runner = Runner {
18778 state,
18779 roles: ResolvedRoles {
18780 implementers: Vec::new(),
18781 judges: Vec::new(),
18782 reviewers: Vec::new(),
18783 fixer: None,
18784 conductor: conductor(),
18785 implementer_roster: Vec::new(),
18786 judge_roster: Vec::new(),
18787 reviewer_roster: Vec::new(),
18788 },
18789 sem: Arc::new(Semaphore::new(1)),
18790 pause: Pause::new(),
18791 interrupt: Pause::new(),
18792 };
18793 let shell = runner.state.config.shell();
18794 runner
18795 .stop_reviewing("round budget spent", &shell, &repo)
18796 .await
18797 .expect("stop_reviewing");
18798 assert_eq!(runner.state.status, RunStatus::Gating);
18799 assert_eq!(
18800 runner.state.contested_handoff.is_some(),
18801 expect,
18802 "{severity:?} + {vote:?}"
18803 );
18804 }
18805 }
18806
18807 #[tokio::test]
18820 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18821 crate::run::pin_test_home();
18822 let home = crate::run::home();
18823
18824 let tmp = tempfile::tempdir().expect("tempdir");
18825 let repo = tmp.path().join("repo");
18826 std::fs::create_dir_all(&repo).unwrap();
18827 init_repo(&repo);
18828 let head = crate::git::rev_parse(&repo, "HEAD")
18829 .await
18830 .expect("rev-parse");
18831 let cache_dir = tmp.path().join("target");
18834
18835 let mut config = Config::default();
18836 config.verify.e2e = vec![format!(
18837 "CARGO_TARGET_DIR='{}' test -f README.md",
18838 cache_dir.display()
18839 )];
18840 config.graph.review_rounds = 1;
18841 config.graph.timeout_verify = Some(2);
18844 config.graph.timeout_resource_wait = 2;
18845
18846 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18847 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18848 .expect("no io error acquiring directly")
18849 {
18850 crate::cache::AcquireOutcome::Acquired(g) => g,
18851 crate::cache::AcquireOutcome::Busy(b) => {
18852 panic!("expected the direct acquire to win the lease first: {b:?}")
18853 }
18854 };
18855
18856 let mut state = RunState::new(
18857 repo.clone(),
18858 "main".to_owned(),
18859 head.clone(),
18860 "task".to_owned(),
18861 config,
18862 );
18863 state.candidates = vec![Candidate {
18864 index: 0,
18865 label: 'A',
18866 agent: "alpha".to_owned(),
18867 branch: "does-not-exist".to_owned(),
18868 worktree: repo.clone(),
18869 summary: String::new(),
18870 stat: String::new(),
18871 files: 0,
18872 commits: 0,
18873 empty: false,
18874 failed: None,
18875 verified_noop: None,
18876 duration_ms: 0,
18877 folded: false,
18878 }];
18879 state.tally = Some(Tally {
18880 first_choice: BTreeMap::from([('A', 1)]),
18881 borda: BTreeMap::new(),
18882 winner: 'A',
18883 rankings: 1,
18884 unanimous_initial: true,
18885 deliberated: false,
18886 changed_votes: 0,
18887 unanimous_final: true,
18888 tie_break: None,
18889 judges: 0,
18890 present: 0,
18891 quorum: 0,
18892 met_quorum: true,
18893 uncontested: Some("only candidate A produced a change".to_owned()),
18894 });
18895 state.reviews = vec![ReviewRound {
18899 round: 1,
18900 head: head.clone(),
18901 verified_head: None,
18902 verified_at: None,
18903 reviews: Vec::new(),
18904 e2e: Vec::new(),
18905 fix: None,
18906 blocking: 1,
18907 answered: 1,
18908 expected: 1,
18909 clean: false,
18910 verify_retried: false,
18911 e2e_deferred: true,
18912 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18913 progressed: false,
18914 vote_split: false,
18915 reconsideration: Vec::new(),
18916 verdict: None,
18917 }];
18918
18919 let mut runner = Runner {
18920 state,
18921 roles: ResolvedRoles {
18922 implementers: Vec::new(),
18923 judges: Vec::new(),
18924 reviewers: Vec::new(),
18925 fixer: None,
18926 conductor: conductor(),
18927 implementer_roster: Vec::new(),
18928 judge_roster: Vec::new(),
18929 reviewer_roster: Vec::new(),
18930 },
18931 sem: Arc::new(Semaphore::new(1)),
18932 pause: Pause::new(),
18933 interrupt: Pause::new(),
18934 };
18935
18936 let shell = runner.state.config.shell();
18937 runner
18938 .stop_reviewing("round budget spent", &shell, &repo)
18939 .await
18940 .expect("stop_reviewing");
18941
18942 let last = runner.state.reviews.last().expect("round record");
18943 assert_eq!(
18944 last.e2e_status(),
18945 E2eStatus::ResourceBlocked,
18946 "the shared cache is still held; the attempt must read as blocked, not deferred or \
18947 failed: {last:?}"
18948 );
18949 assert_eq!(
18950 last.verified_head.as_deref(),
18951 Some(head.as_str()),
18952 "which commit this attempt targeted is known even though nothing finished checking \
18953 it"
18954 );
18955 let first_attempt_at = last
18956 .verified_at
18957 .expect("when this attempt ran is known too");
18958 assert_ne!(
18959 runner.state.status,
18960 RunStatus::Blocked,
18961 "contention is evidence about the machine, not the patch — it must not settle the \
18962 run as blocked: {:?}",
18963 runner.state.status
18964 );
18965 assert!(
18966 !runner
18967 .state
18968 .events
18969 .iter()
18970 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18971 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18972 runner.state.events
18973 );
18974
18975 runner
18980 .stop_reviewing("round budget spent", &shell, &repo)
18981 .await
18982 .expect("stop_reviewing retry");
18983 assert_eq!(
18984 runner.state.reviews.len(),
18985 1,
18986 "no new round was started: {:?}",
18987 runner.state.reviews
18988 );
18989 let last = runner.state.reviews.last().expect("round record");
18990 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18991 assert!(
18992 last.verified_at.expect("still known") > first_attempt_at,
18993 "a second reentry must be a fresh attempt, not a stale copy of the first"
18994 );
18995 assert_ne!(runner.state.status, RunStatus::Blocked);
18996
18997 held.release();
18998 }
18999
19000 #[cfg(unix)]
19001 #[tokio::test]
19002 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
19003 ask_test_home();
19004 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
19005 let tmp = tempfile::tempdir().unwrap();
19006 let repo = tmp.path().join("repo");
19007 std::fs::create_dir(&repo).unwrap();
19008 init_repo(&repo);
19009 let run_git = |args: &[&str]| {
19010 let out = std::process::Command::new("git")
19011 .args(args)
19012 .current_dir(&repo)
19013 .output()
19014 .unwrap();
19015 assert!(
19016 out.status.success(),
19017 "{args:?}: {}",
19018 String::from_utf8_lossy(&out.stderr)
19019 );
19020 };
19021 let origin = tmp.path().join("origin.git");
19022 run_git(&["init", "--bare", origin.to_str().unwrap()]);
19023 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
19024 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
19025 run_git(&["add", "spec.rs"]);
19026 run_git(&["commit", "-m", "initial spec"]);
19027 run_git(&["push", "origin", "main"]);
19028 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
19029 run_git(&["checkout", "-b", "fix/review"]);
19030 std::fs::write(
19031 repo.join("check.rs"),
19032 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
19033 )
19034 .unwrap();
19035 run_git(&["add", "check.rs"]);
19036 run_git(&["commit", "-m", "use spec"]);
19037 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
19038 run_git(&["checkout", "main"]);
19039 std::fs::write(
19040 repo.join("spec.rs"),
19041 "struct AgentSpec { max_turn_secs: u64 }\n",
19042 )
19043 .unwrap();
19044 run_git(&["add", "spec.rs"]);
19045 run_git(&["commit", "-m", "extend spec"]);
19046 run_git(&["push", "origin", "main"]);
19047 run_git(&["checkout", "fix/review"]);
19048
19049 let mut runner = runner_at(RunStatus::Reviewing);
19050 runner.state.repo = repo.clone();
19051 runner.state.base_commit = base;
19052 runner.state.pr_head = Some("fix/review".into());
19053 runner.state.config.graph.review_rounds = max_rounds;
19054 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
19055 runner.state.config.graph.sessions = false;
19056 runner.state.config.disk.min_free_bytes = 0;
19057 runner.state.config.verify.e2e =
19058 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
19059 runner.state.candidates = vec![Candidate {
19060 index: 0,
19061 label: 'A',
19062 agent: "fixture".into(),
19063 branch: "fix/review".into(),
19064 worktree: repo.clone(),
19065 summary: String::new(),
19066 stat: String::new(),
19067 files: 1,
19068 commits: 1,
19069 empty: false,
19070 failed: None,
19071 verified_noop: None,
19072 duration_ms: 0,
19073 folded: false,
19074 }];
19075 runner.state.tally = Some(Tally {
19076 first_choice: BTreeMap::new(),
19077 borda: BTreeMap::new(),
19078 winner: 'A',
19079 rankings: 0,
19080 unanimous_initial: true,
19081 deliberated: false,
19082 changed_votes: 0,
19083 unanimous_final: true,
19084 tie_break: None,
19085 judges: 0,
19086 present: 0,
19087 quorum: 0,
19088 met_quorum: true,
19089 uncontested: Some("review-only".into()),
19090 });
19091 let mut last = review_round(false, 0, 1, 1, false, false);
19092 last.head = reviewed.clone();
19093 last.e2e_deferred = true;
19094 last.e2e[0].resource_blocked = true;
19095 last.e2e[0].output_tail = "waiting for the shared build cache".into();
19096 if max_rounds == 1 {
19097 last.e2e.clear();
19098 }
19099 runner.state.reviews = vec![last];
19100 let calls = tmp.path().join("fix-calls");
19101 let mut fixer = spec("repair");
19102 fixer
19103 .env
19104 .insert("FIX_CALLS".into(), calls.display().to_string());
19105 fixer.command = vec!["sh".into(), "-c".into(), r#"
19106 echo fix >> "$FIX_CALLS"
19107 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
19108 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
19109 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
19110 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
19111 "#.into()];
19112 let mut reviewer = spec("reviewer");
19113 reviewer.command = vec![
19114 "sh".into(),
19115 "-c".into(),
19116 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
19117 .into(),
19118 ];
19119 runner.roles.fixer = Some(vec![fixer]);
19120 runner.roles.reviewers = vec![reviewer];
19121
19122 runner.sync_to_base().await.unwrap();
19123 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
19124 assert_ne!(rebased, reviewed);
19125 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
19126 if reentry {
19127 let shell = runner.state.config.shell();
19130 runner
19131 .stop_unresolved_review("deferred verification", &shell, &repo)
19132 .await
19133 .unwrap();
19134 assert!(runner.catchup_repairable());
19135 assert!(!calls.exists());
19136 runner.state = RunState::load(&runner.state.id).unwrap();
19137 assert!(runner.catchup_repairable());
19138 }
19139 runner.review_loop().await.unwrap();
19140 let first = &runner.state.reviews[0];
19141 assert_eq!(first.head, reviewed);
19142 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
19143 assert!(
19144 first.verify_retried,
19145 "a reproduced build failure gets exactly one retry: {:?}",
19146 first.e2e
19147 );
19148 assert!(first.e2e[0].output_tail.contains("E0063"));
19149 assert!(!runner.catchup_repairable());
19150 if max_rounds == 2 {
19151 assert_eq!(
19152 runner.state.status,
19153 RunStatus::Gating,
19154 "{}",
19155 crate::report::run(&runner.state)
19156 );
19157 assert_eq!(runner.state.reviews.len(), 2);
19158 assert!(first.fix.as_ref().unwrap().committed);
19159 assert!(runner.state.reviews[1].clean);
19160 assert_eq!(
19161 runner
19162 .state
19163 .reviews
19164 .iter()
19165 .filter(|r| r.fix.is_some())
19166 .count(),
19167 1
19168 );
19169 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
19170 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
19171 } else {
19172 assert_eq!(runner.state.status, RunStatus::Blocked);
19173 assert!(first.fix.is_none());
19174 assert!(!calls.exists());
19175 let recovery = crate::recovery::assess(
19176 "prior",
19177 Some(&runner.state),
19178 &runner.landing_base(),
19179 crate::recovery::Relation::Diverged,
19180 );
19181 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
19182 let reasons = recovery.reasons.join("\n");
19183 assert!(
19184 reasons.contains(&reviewed) && reasons.contains(&rebased),
19185 "{reasons}"
19186 );
19187 assert!(
19188 reasons.contains("base sync to")
19189 && reasons.contains("additional review is required"),
19190 "{reasons}"
19191 );
19192 }
19193 }
19194 }
19195
19196 #[tokio::test]
19208 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
19209 crate::run::pin_test_home();
19210 let home = crate::run::home();
19211
19212 let tmp = tempfile::tempdir().expect("tempdir");
19213 let repo = tmp.path().join("repo");
19214 std::fs::create_dir_all(&repo).unwrap();
19215 init_repo(&repo);
19216 let head = crate::git::rev_parse(&repo, "HEAD")
19217 .await
19218 .expect("rev-parse");
19219 let cache_dir = tmp.path().join("target");
19220
19221 let mut config = Config::default();
19222 config.verify.e2e = vec![format!(
19223 "CARGO_TARGET_DIR='{}' test -f README.md",
19224 cache_dir.display()
19225 )];
19226 config.graph.review_rounds = 1;
19227 config.graph.timeout_verify = Some(2);
19228 config.graph.timeout_resource_wait = 2;
19229
19230 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19231 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19232 .expect("no io error acquiring directly")
19233 {
19234 crate::cache::AcquireOutcome::Acquired(g) => g,
19235 crate::cache::AcquireOutcome::Busy(b) => {
19236 panic!("expected the direct acquire to win the lease first: {b:?}")
19237 }
19238 };
19239
19240 let mut state = RunState::new(
19241 repo.clone(),
19242 "main".to_owned(),
19243 head.clone(),
19244 "task".to_owned(),
19245 config,
19246 );
19247 state.candidates = vec![Candidate {
19248 index: 0,
19249 label: 'A',
19250 agent: "alpha".to_owned(),
19251 branch: "does-not-exist".to_owned(),
19252 worktree: repo.clone(),
19253 summary: String::new(),
19254 stat: String::new(),
19255 files: 0,
19256 commits: 0,
19257 empty: false,
19258 failed: None,
19259 verified_noop: None,
19260 duration_ms: 0,
19261 folded: false,
19262 }];
19263 state.tally = Some(Tally {
19264 first_choice: BTreeMap::from([('A', 1)]),
19265 borda: BTreeMap::new(),
19266 winner: 'A',
19267 rankings: 1,
19268 unanimous_initial: true,
19269 deliberated: false,
19270 changed_votes: 0,
19271 unanimous_final: true,
19272 tie_break: None,
19273 judges: 0,
19274 present: 0,
19275 quorum: 0,
19276 met_quorum: true,
19277 uncontested: Some("only candidate A produced a change".to_owned()),
19278 });
19279 state.reviews = vec![ReviewRound {
19283 round: 1,
19284 head: head.clone(),
19285 verified_head: Some(head.clone()),
19286 verified_at: Some(jiff::Timestamp::now()),
19287 reviews: Vec::new(),
19288 e2e: vec![CommandOutcome {
19289 command: format!(
19290 "CARGO_TARGET_DIR='{}' test -f README.md",
19291 cache_dir.display()
19292 ),
19293 code: None,
19294 output_tail: "waiting for the shared build cache".to_owned(),
19295 duration_ms: 0,
19296 resource_blocked: true,
19297 }],
19298 fix: None,
19299 blocking: 1,
19300 answered: 1,
19301 expected: 1,
19302 clean: false,
19303 verify_retried: false,
19304 e2e_deferred: false,
19305 e2e_defer_reason: None,
19306 progressed: false,
19307 vote_split: false,
19308 reconsideration: Vec::new(),
19309 verdict: None,
19310 }];
19311
19312 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
19313 let mut runner = Runner {
19314 state,
19315 roles: ResolvedRoles {
19316 implementers: Vec::new(),
19317 judges: Vec::new(),
19318 reviewers: Vec::new(),
19319 fixer: None,
19320 conductor: conductor(),
19321 implementer_roster: Vec::new(),
19322 judge_roster: Vec::new(),
19323 reviewer_roster: Vec::new(),
19324 },
19325 sem: Arc::new(Semaphore::new(1)),
19326 pause: Pause::new(),
19327 interrupt: Pause::new(),
19328 };
19329
19330 runner.review_loop().await.expect("review_loop");
19335
19336 assert_eq!(
19337 runner.state.reviews.len(),
19338 1,
19339 "no new round was started on top of the unresolved one: {:?}",
19340 runner.state.reviews
19341 );
19342 let last = &runner.state.reviews[0];
19343 assert_eq!(
19344 last.e2e_status(),
19345 E2eStatus::ResourceBlocked,
19346 "still contended: {last:?}"
19347 );
19348 assert!(
19349 last.verified_at.expect("still known") > first_attempt_at,
19350 "review_loop must have actually retried the check, not left it exactly as found"
19351 );
19352 assert_ne!(
19353 runner.state.status,
19354 RunStatus::Blocked,
19355 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
19356 runner.state.status
19357 );
19358
19359 held.release();
19360 }
19361
19362 #[tokio::test]
19363 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
19364 crate::run::pin_test_home();
19365 for status in [RunStatus::Blocked, RunStatus::Gating] {
19366 let mut runner = runner_at(status);
19367 runner.state.config.graph.land = true;
19368 runner.state.gate_ran = true;
19369 runner.state.merge = Some(MergeOutcome {
19370 mode: MergeMode::Pr,
19371 ok: false,
19372 detail: "the fixer produced no commit".into(),
19373 empty: false,
19374 });
19375 let url = "https://example.invalid/o/r/pull/1";
19376 runner.state.pr = Some(crate::run::PrRecord {
19377 url: url.into(),
19378 number: 1,
19379 state: "open".into(),
19380 checks: "red".into(),
19381 round: 1,
19382 rounds: 1,
19383 red_at_merge: Vec::new(),
19384 });
19385 runner.state.save().unwrap();
19386 runner.execute_graph().await.unwrap();
19390 assert_eq!(runner.state.status, RunStatus::Blocked);
19391 assert!(runner.state.land_sync_pending);
19392 assert!(runner.state.events.iter().any(|event|
19393 event.node == "land" && event.message == format!("watching {url}")));
19394 assert_eq!(
19395 runner.state.merge.as_ref().unwrap().detail,
19396 "the fixer produced no commit"
19397 );
19398 }
19399 }
19400
19401 #[tokio::test]
19402 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
19403 crate::run::pin_test_home();
19404 let tmp = tempfile::tempdir().expect("tempdir");
19405 let repo = tmp.path().join("repo");
19406 std::fs::create_dir_all(&repo).unwrap();
19407 init_repo(&repo);
19408
19409 let mut config = Config::default();
19410 config.merge.mode = MergeMode::Pr;
19411 config.graph.land = true;
19412 config.graph.land_approval = false;
19413
19414 let mut state = RunState::new(
19415 repo.clone(),
19416 "main".to_owned(),
19417 "deadbeef".to_owned(),
19418 "task".to_owned(),
19419 config,
19420 );
19421 state.candidates = vec![Candidate {
19422 index: 0,
19423 label: 'A',
19424 agent: "alpha".to_owned(),
19425 branch: "does-not-exist".to_owned(),
19426 worktree: repo.clone(),
19427 summary: String::new(),
19428 stat: String::new(),
19429 files: 0,
19430 commits: 0,
19431 empty: false,
19432 failed: None,
19433 verified_noop: None,
19434 duration_ms: 0,
19435 folded: false,
19436 }];
19437 state.tally = Some(Tally {
19438 first_choice: BTreeMap::from([('A', 1)]),
19439 borda: BTreeMap::new(),
19440 winner: 'A',
19441 rankings: 1,
19442 unanimous_initial: true,
19443 deliberated: false,
19444 changed_votes: 0,
19445 unanimous_final: true,
19446 tie_break: None,
19447 judges: 0,
19448 present: 0,
19449 quorum: 0,
19450 met_quorum: true,
19451 uncontested: Some("only candidate A produced a change".to_owned()),
19452 });
19453 state.reviews = vec![ReviewRound {
19454 round: 1,
19455 head: "deadbeef".to_owned(),
19456 verified_head: None,
19457 verified_at: None,
19458 reviews: Vec::new(),
19459 e2e: Vec::new(),
19460 fix: None,
19461 blocking: 0,
19462 answered: 0,
19463 expected: 0,
19464 clean: true,
19465 verify_retried: false,
19466 e2e_deferred: false,
19467 e2e_defer_reason: None,
19468 progressed: false,
19469 vote_split: false,
19470 reconsideration: Vec::new(),
19471 verdict: None,
19472 }];
19473 state.gate = vec![CommandOutcome {
19474 command: "test".to_owned(),
19475 code: Some(0),
19476 output_tail: String::new(),
19477 duration_ms: 0,
19478 resource_blocked: false,
19479 }];
19480 state.gate_ran = true;
19481 state.status = RunStatus::Landing;
19485 state.merge = Some(MergeOutcome {
19486 mode: MergeMode::Pr,
19487 ok: true,
19488 detail: "https://example.invalid/x/y/pull/1".to_owned(),
19489 empty: false,
19490 });
19491
19492 ask_test_home();
19496 state.save().expect("save landing run");
19499 let store = ask::Questions::open();
19500 let q = ask_open_question(&store, &state.id);
19501
19502 let mut runner = Runner {
19503 state,
19504 roles: ResolvedRoles {
19505 implementers: Vec::new(),
19506 judges: Vec::new(),
19507 reviewers: Vec::new(),
19508 fixer: None,
19509 conductor: conductor(),
19510 implementer_roster: Vec::new(),
19511 judge_roster: Vec::new(),
19512 reviewer_roster: Vec::new(),
19513 },
19514 sem: Arc::new(Semaphore::new(1)),
19515 pause: Pause::new(),
19516 interrupt: Pause::new(),
19517 };
19518
19519 runner.execute().await.expect("execute");
19524
19525 assert_eq!(
19526 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19527 Some("https://example.invalid/x/y/pull/1"),
19528 "reentry must not push again or open a second pull request over the \
19529 one `land` is already watching"
19530 );
19531 assert_ne!(
19532 runner.state.status,
19533 RunStatus::Landing,
19534 "land could not actually reach the fake pull request, so it must \
19535 have given up rather than left the run silently parked forever"
19536 );
19537 assert_eq!(runner.state.status, RunStatus::Blocked);
19541 assert!(
19542 store.get(&q.id).unwrap().status.open(),
19543 "Blocked is still alive; settle_questions must have been a no-op here"
19544 );
19545 }
19546
19547 fn state_with_round(round: ReviewRound) -> RunState {
19548 let mut s = RunState::new(
19549 PathBuf::from("/repo"),
19550 "main".to_owned(),
19551 "abc1234".to_owned(),
19552 "add retries".to_owned(),
19553 Config::default(),
19554 );
19555 s.reviews = vec![round];
19556 s
19557 }
19558
19559 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19560 crate::verdict::Finding {
19561 id: id.to_owned(),
19562 severity,
19563 file: None,
19564 line: None,
19565 title: title.to_owned(),
19566 detail: String::new(),
19567 }
19568 }
19569
19570 #[test]
19571 fn pr_body_names_open_findings_and_declined_ones() {
19572 let round = ReviewRound {
19573 round: 2,
19574 head: "deadbee".to_owned(),
19575 verified_head: None,
19576 verified_at: None,
19577 reviews: vec![ReviewRecord {
19578 attempts: 0,
19579 reviewer: 1,
19580 agent: "alpha".to_owned(),
19581 summary: String::new(),
19582 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19583 vote: None,
19584 failed: None,
19585 duration_ms: 0,
19586 }],
19587 e2e: vec![CommandOutcome {
19588 command: "cargo test".to_owned(),
19589 code: Some(0),
19590 output_tail: String::new(),
19591 duration_ms: 0,
19592 resource_blocked: false,
19593 }],
19594 verify_retried: false,
19595 e2e_deferred: false,
19596 e2e_defer_reason: None,
19597 fix: Some(FixRecord {
19598 agent: "alpha".to_owned(),
19599 addressed: Vec::new(),
19600 rejected: vec![crate::verdict::Rejection {
19601 id: "R1-1-1".to_owned(),
19602 why: "not reachable from any caller".to_owned(),
19603 }],
19604 notes: String::new(),
19605 committed: true,
19606 failed: None,
19607 duration_ms: 0,
19608 continuation: None,
19609 }),
19610 blocking: 0,
19611 answered: 1,
19612 expected: 1,
19613 clean: false,
19614 progressed: true,
19615 vote_split: false,
19616 reconsideration: Vec::new(),
19617 verdict: None,
19618 };
19619 let state = state_with_round(round);
19620 let body = pr_message(&state, 'A').body;
19621
19622 assert!(body.contains("add retries"), "the task must still be there");
19623 assert!(body.contains("R2-1-1"), "{body}");
19624 assert!(!body.contains("unused import"), "{body}");
19625 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19626 assert!(
19627 !body.contains("not reachable from any caller"),
19628 "the reason it was declined: {body}"
19629 );
19630 }
19631
19632 #[test]
19633 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19634 let round = ReviewRound {
19635 round: 1,
19636 head: "deadbee".to_owned(),
19637 verified_head: None,
19638 verified_at: None,
19639 reviews: vec![ReviewRecord {
19640 attempts: 0,
19641 reviewer: 1,
19642 agent: "alpha".to_owned(),
19643 summary: String::new(),
19644 findings: Vec::new(),
19645 vote: None,
19646 failed: None,
19647 duration_ms: 0,
19648 }],
19649 e2e: Vec::new(),
19650 verify_retried: false,
19651 e2e_deferred: false,
19652 e2e_defer_reason: None,
19653 fix: None,
19654 blocking: 0,
19655 answered: 1,
19656 expected: 1,
19657 clean: true,
19658 progressed: false,
19659 vote_split: false,
19660 reconsideration: Vec::new(),
19661 verdict: None,
19662 };
19663 let state = state_with_round(round);
19664 let body = pr_message(&state, 'A').body;
19665 assert!(!body.contains("Open review findings"), "{body}");
19666 assert!(!body.contains("Declined"), "{body}");
19667 }
19668
19669 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19670 let mut state = RunState::new(
19671 PathBuf::from("/repo"),
19672 "main".to_owned(),
19673 "abc1234".to_owned(),
19674 instruction.to_owned(),
19675 Config::default(),
19676 );
19677 state.candidates.push(Candidate {
19678 index: 0,
19679 label: 'A',
19680 agent: "alpha".to_owned(),
19681 branch: "magi/x/A".to_owned(),
19682 worktree: PathBuf::from("/wt"),
19683 summary: summary.to_owned(),
19684 stat: String::new(),
19685 files: 1,
19686 commits: 1,
19687 empty: false,
19688 failed: None,
19689 verified_noop: None,
19690 folded: false,
19691 duration_ms: 0,
19692 });
19693 state
19694 }
19695
19696 fn review_state(subjects: &[&str]) -> RunState {
19697 let mut state = state_with_summary(
19698 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19699 "",
19700 );
19701 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19702 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19703 state
19704 }
19705
19706 #[tokio::test]
19709 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19710 ask_test_home();
19711 let tmp = tempfile::tempdir().unwrap();
19712 let repo = tmp.path().join("repo");
19713 std::fs::create_dir_all(&repo).unwrap();
19714 init_repo(&repo);
19715 let origin = tmp.path().join("origin.git");
19716 let g = |dir: &Path, args: &[&str]| {
19717 let out = std::process::Command::new("git")
19718 .args(args)
19719 .current_dir(dir)
19720 .quiet()
19721 .output()
19722 .expect("spawn git");
19723 assert!(
19724 out.status.success(),
19725 "git {args:?}: {}",
19726 String::from_utf8_lossy(&out.stderr)
19727 );
19728 };
19729 g(
19730 tmp.path(),
19731 &[
19732 "clone",
19733 "--bare",
19734 "-q",
19735 repo.to_str().unwrap(),
19736 origin.to_str().unwrap(),
19737 ],
19738 );
19739 g(
19740 &repo,
19741 &["remote", "add", "origin", origin.to_str().unwrap()],
19742 );
19743 let c1 = git::rev_parse(&repo, "main").await.unwrap();
19744
19745 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19747 g(&repo, &["add", "-A"]);
19748 g(
19749 &repo,
19750 &["commit", "-q", "-m", "chore(deps): update a crate"],
19751 );
19752 g(&repo, &["push", "-q", "origin", "main"]);
19753 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19754 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19755 g(&repo, &["add", "-A"]);
19756 g(
19757 &repo,
19758 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19759 );
19760 g(&repo, &["checkout", "-q", "main"]);
19761
19762 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19763 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19764 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19766 assert_eq!(fallback, c1);
19767
19768 let mut state = review_state(&[
19769 "chore(deps): update a crate",
19770 "fix(daemon): apply a chosen action",
19771 ]);
19772 state.repo = repo.clone();
19773 state.base_branch = "main".to_owned();
19774 state.base_commit = c1;
19775 refresh_reviewed_commits(&mut state, "feat/own").await;
19776 assert_eq!(
19777 state.reviewed_commits,
19778 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19779 );
19780 assert_eq!(
19781 review_title(&state).as_deref(),
19782 Some("fix(daemon): apply a chosen action")
19783 );
19784 assert_eq!(
19785 leaked_subjects(&state, "feat/own").await,
19786 Some(vec!["chore(deps): update a crate".to_owned()])
19787 );
19788 state.config.merge.remote = "nowhere".to_owned();
19791 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19792 }
19793
19794 struct SummaryFixture {
19798 _tmp: tempfile::TempDir,
19799 repo: PathBuf,
19800 reply: PathBuf,
19801 runner: Runner,
19802 }
19803
19804 fn summary_fixture() -> SummaryFixture {
19805 ask_test_home();
19806 let tmp = tempfile::tempdir().unwrap();
19807 let repo = tmp.path().join("repo");
19808 std::fs::create_dir_all(&repo).unwrap();
19809 init_repo(&repo);
19810 let origin = tmp.path().join("origin.git");
19811 let g = |args: &[&str]| {
19812 let out = std::process::Command::new("git")
19813 .args(args)
19814 .current_dir(&repo)
19815 .quiet()
19816 .output()
19817 .expect("spawn git");
19818 assert!(
19819 out.status.success(),
19820 "git {args:?}: {}",
19821 String::from_utf8_lossy(&out.stderr)
19822 );
19823 };
19824 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19825 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19826 g(&["push", "-q", "origin", "main"]);
19827 g(&["checkout", "-q", "-b", "feat/own"]);
19828 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19829 g(&["add", "-A"]);
19830 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19831 let reply = tmp.path().join("reply.txt");
19832 std::fs::write(
19833 &reply,
19834 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19835 )
19836 .unwrap();
19837 let mut runner = runner_at(RunStatus::Gating);
19838 let mut state = review_state(&["feat(own): add the own file"]);
19839 state.repo = repo.clone();
19840 state.base_branch = "main".to_owned();
19841 state.candidates[0].branch = "feat/own".to_owned();
19842 state.candidates[0].worktree = repo.clone();
19843 let mut author = spec("alpha");
19844 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19845 author
19846 .env
19847 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19848 state.config.agents = vec![author];
19849 runner.state = state;
19850 SummaryFixture {
19851 _tmp: tmp,
19852 repo,
19853 reply,
19854 runner,
19855 }
19856 }
19857
19858 async fn posted(f: &mut SummaryFixture) -> PrMessage {
19859 let winner = f.runner.state.candidates[0].clone();
19860 f.runner
19861 .guarded_pr_message(&winner, None, true)
19862 .await
19863 .unwrap()
19864 .expect("an English description never parks")
19865 }
19866
19867 fn clean_round_with_minor_findings() -> ReviewRound {
19868 let mut round = passing_round(2, None);
19869 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19870 round.reviews = vec![ReviewRecord {
19871 attempts: 0,
19872 reviewer: 1,
19873 agent: "alpha".into(),
19874 summary: "内部レビュー".into(),
19875 findings: vec![
19876 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19877 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19878 ],
19879 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19880 failed: None,
19881 duration_ms: 0,
19882 }];
19883 round
19884 }
19885
19886 #[tokio::test]
19887 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19888 for review_only in [false, true] {
19889 let mut f = summary_fixture();
19890 if !review_only {
19891 f.runner.state.candidates[0].agent = "alpha".into();
19892 }
19893 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19894 assert!(f.runner.state.reviews[0].clean);
19895 assert!(f.runner.state.reviews[0].fix.is_none());
19896 assert!(f.runner.state.open_findings().is_empty());
19897 assert_eq!(
19898 crate::pr_outcome::required_finding_ids(&f.runner.state),
19899 ["R2-1-1", "R2-1-2"]
19900 );
19901 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19902 let evidence = crate::pr_outcome::summary_evidence(
19903 &f.runner.state,
19904 crate::pr_outcome::FinalHead::At(&head),
19905 );
19906 assert!(
19907 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19908 );
19909 let omitted = posted(&mut f).await;
19911 assert!(omitted.body.contains("No description"));
19912 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19913 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19914 assert!(f.runner.state.change_summary.is_none());
19915 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19916 let described = posted(&mut f).await;
19917 assert!(described.body.contains("improve documentation for callers"));
19918 assert!(described.body.contains("remaining edge case"));
19919 assert!(described.body.contains("## Open review findings"));
19920 assert!(!described.body.contains("未修正"));
19921 }
19922 }
19923
19924 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19925 let mut f = summary_fixture();
19926 if !review_only {
19927 f.runner.state.candidates[0].agent = "alpha".into();
19928 }
19929 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19930 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19931 fixed
19932 .fix
19933 .as_mut()
19934 .unwrap()
19935 .rejected
19936 .push(crate::verdict::Rejection {
19937 id: "R1-1-1".into(),
19938 why: "呼び出されないコードのため辞退".into(),
19939 });
19940 let mut clean = clean_round_with_minor_findings();
19941 clean.verified_head = Some(head.clone());
19942 f.runner.state.reviews = vec![fixed, clean];
19943 ok_gate(&mut f.runner.state, Some(&head));
19944 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19945 f.runner.state.config.agents.push(scripted("beta", reply));
19946 f.runner.state.config.roles.language_judge =
19947 Some(crate::config::AgentChoice::One("beta".into()));
19948 f
19949 }
19950
19951 #[tokio::test]
19952 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19953 for review_only in [false, true] {
19954 let mut f = rewrite_fixture_with_findings(review_only,
19955 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19956 let m = posted(&mut f).await;
19957 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19958 let expected =
19959 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19960 assert!(m.body.contains(&expected));
19961 assert!(m.body.contains("## Review rounds"));
19962 assert!(
19963 m.body
19964 .contains("Gate (`verify.gate`): passed on the final commit")
19965 );
19966 assert!(
19967 m.body
19968 .contains("Tests (`verify.e2e`): passed on the final commit")
19969 );
19970 let c = f.runner.state.change_summary.as_ref().unwrap();
19971 assert!(
19972 m.body
19973 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19974 );
19975 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19976 assert!(crate::pr_outcome::covers_required_findings(
19977 &f.runner.state,
19978 &m.body
19979 ));
19980 assert!(m.body.contains("documentation remains incomplete"));
19981 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19982 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19983 assert!(f.runner.state.github_text.is_none());
19984 }
19985 }
19986
19987 #[tokio::test]
19988 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19989 for review_only in [false, true] {
19990 for reply in [
19991 "TITLE: fix: public description\nA short English description.",
19992 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19993 ] {
19994 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19995 let winner = f.runner.state.candidates[0].clone();
19996 let posted = f
19997 .runner
19998 .guarded_pr_message(&winner, None, true)
19999 .await
20000 .unwrap();
20001 assert!(posted.is_none());
20002 assert!(f.runner.state.parked);
20003 let withheld = f.runner.state.github_text.as_ref().unwrap();
20004 assert!(
20005 withheld.rewrite.is_none(),
20006 "an incomplete rewrite must not be selectable"
20007 );
20008 let q = ask::Questions::open()
20009 .get(withheld.question.as_ref().unwrap())
20010 .unwrap();
20011 assert!(
20012 !q.choices
20013 .contains(&crate::github_text::USE_REWRITE.to_owned())
20014 );
20015 assert!(
20016 q.detail.contains("R1-1-1")
20017 && q.detail.contains("R2-1-1")
20018 && q.detail.contains("R2-1-2")
20019 );
20020 }
20021 }
20022 }
20023
20024 #[tokio::test]
20025 async fn japanese_reports_stay_local_in_both_public_run_modes() {
20026 for review_only in [false, true] {
20027 let mut f = summary_fixture();
20028 if !review_only {
20029 f.runner.state.candidates[0].agent = "alpha".into();
20030 }
20031 f.runner.state.instruction = "内部タスクの説明".into();
20032 f.runner.state.candidates[0].summary = "内部の候補報告".into();
20033 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20034 f.runner.state.reviews = vec![
20035 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
20036 round_with_notes(2, Some("二回目の内部修正報告です。")),
20037 passing_round(3, Some(&head)),
20038 ];
20039 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
20040 ok_gate(&mut f.runner.state, Some(&head));
20041 let captured = f.repo.join("public-prompt.txt");
20042 let author = &mut f.runner.state.config.agents[0];
20043 author.command = vec![
20044 "sh".into(),
20045 "-c".into(),
20046 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20047 ];
20048 author.env.insert(
20049 "PROMPT_FILE".into(),
20050 captured.to_string_lossy().into_owned(),
20051 );
20052 let m = posted(&mut f).await;
20053 assert!(m.body.contains("Adds `own.txt`"));
20054 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
20055 assert!(m.body.contains("R1-1-1"));
20056 assert!(
20057 m.body
20058 .contains("Gate (`verify.gate`): passed on the final commit")
20059 );
20060 assert!(
20061 m.body
20062 .contains("Tests (`verify.e2e`): passed on the final commit")
20063 );
20064 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20065 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
20066 let prompt = std::fs::read_to_string(captured).unwrap();
20067 assert!(prompt.contains(&format!(
20068 "git diff {} {head}",
20069 f.runner.state.change_summary.as_ref().unwrap().base
20070 )));
20071 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
20072 assert_eq!(
20073 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
20074 "二回目の内部修正報告です。"
20075 );
20076 }
20077 }
20078
20079 #[tokio::test]
20080 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
20081 for review_only in [false, true] {
20082 let mut f = summary_fixture();
20083 if !review_only {
20084 f.runner.state.candidates[0].agent = "alpha".into();
20085 }
20086 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
20087 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
20088 std::fs::write(&f.reply, "no usable description").unwrap();
20089 let failed = posted(&mut f).await;
20090 assert!(
20091 failed
20092 .body
20093 .contains("No description of what this change does could be generated")
20094 );
20095 assert!(failed.body.is_ascii());
20096 assert!(f.runner.state.github_text.is_none());
20097 assert!(f.runner.state.change_summary.is_none());
20098 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20099 let evidence = crate::pr_outcome::summary_evidence(
20100 &f.runner.state,
20101 crate::pr_outcome::FinalHead::At(&head),
20102 );
20103 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
20104 assert!(
20105 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
20106 .exists()
20107 );
20108 f.runner.state =
20109 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
20110 std::fs::write(
20111 &f.reply,
20112 "TITLE: feat(own): add the own file\nAdds the committed own file.",
20113 )
20114 .unwrap();
20115 let retried = posted(&mut f).await;
20116 assert!(retried.body.contains("Adds the committed own file."));
20117 assert_eq!(
20118 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
20119 "日本語の修正報告"
20120 );
20121 }
20122 }
20123
20124 #[tokio::test]
20125 async fn same_head_verification_and_findings_invalidate_public_prose() {
20126 let mut f = summary_fixture();
20127 posted(&mut f).await;
20128 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20129 let original = f
20130 .runner
20131 .state
20132 .change_summary
20133 .as_ref()
20134 .unwrap()
20135 .evidence
20136 .clone();
20137 ok_gate(&mut f.runner.state, Some(&head));
20138 std::fs::write(
20139 &f.reply,
20140 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
20141 )
20142 .unwrap();
20143 let verified = posted(&mut f).await;
20144 assert!(verified.body.contains("now verified"));
20145 assert_ne!(
20146 f.runner.state.change_summary.as_ref().unwrap().evidence,
20147 original
20148 );
20149 let mut round = round_with_notes(1, Some("内部報告"));
20150 round.reviews.push(ReviewRecord {
20151 attempts: 0,
20152 reviewer: 1,
20153 agent: "alpha".into(),
20154 summary: "内部レビュー".into(),
20155 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
20156 vote: None,
20157 failed: None,
20158 duration_ms: 0,
20159 });
20160 round
20161 .fix
20162 .as_mut()
20163 .unwrap()
20164 .rejected
20165 .push(crate::verdict::Rejection {
20166 id: "R1-1-1".into(),
20167 why: "呼び出し元から到達できない".into(),
20168 });
20169 f.runner.state.reviews = vec![round];
20170 let omitted = posted(&mut f).await;
20172 assert!(omitted.body.contains("No description"));
20173 assert!(omitted.body.contains("R1-1-1"));
20174 assert!(!omitted.body.contains("到達"));
20175 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
20176 let described = posted(&mut f).await;
20177 assert!(described.body.contains("unreachable from callers"));
20178 assert!(described.body.contains("R1-1-1"));
20179 assert!(!described.body.contains("到達"));
20180 let evidence = f
20181 .runner
20182 .state
20183 .change_summary
20184 .as_ref()
20185 .unwrap()
20186 .evidence
20187 .clone();
20188 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
20189 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
20190 assert!(posted(&mut f).await.body.contains("rationale updated"));
20191 assert_ne!(
20192 f.runner.state.change_summary.as_ref().unwrap().evidence,
20193 evidence
20194 );
20195 f.runner
20197 .state
20198 .change_summary
20199 .as_mut()
20200 .unwrap()
20201 .evidence
20202 .clear();
20203 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
20204 assert!(posted(&mut f).await.body.contains("current specification"));
20205 }
20206
20207 #[tokio::test]
20208 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
20209 let mut f = summary_fixture();
20210 let m = posted(&mut f).await;
20211 assert_eq!(m.title, "feat(own): add the own file");
20212 assert!(
20213 m.body
20214 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
20215 "{}",
20216 m.body
20217 );
20218 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
20219 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
20220 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20221 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20222 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
20223 let kept = f.runner.state.change_summary.as_ref().unwrap();
20224 assert_eq!(
20225 (kept.head.as_str(), kept.base.as_str()),
20226 (head.as_str(), base.as_str())
20227 );
20228 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
20229 }
20230
20231 #[tokio::test]
20232 async fn review_only_summary_receives_the_configured_github_language() {
20233 let mut f = summary_fixture();
20234 f.runner.state.config.graph.github_language = "ja".into();
20235 let captured = f.repo.join("summary-prompt.txt");
20236 let author = &mut f.runner.state.config.agents[0];
20237 author.command = vec![
20238 "sh".into(),
20239 "-c".into(),
20240 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20241 ];
20242 author.env.insert(
20243 "PROMPT_FILE".into(),
20244 captured.to_string_lossy().into_owned(),
20245 );
20246 std::fs::write(
20247 &f.reply,
20248 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
20249 )
20250 .unwrap();
20251 let m = posted(&mut f).await;
20252 let prompt = std::fs::read_to_string(captured).unwrap();
20253 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
20254 assert!(!prompt.contains("in English"));
20255 assert_eq!(m.title, "feat(own): ファイルを追加");
20256 assert!(m.body.contains("own.txt を追加します。"));
20257 }
20258
20259 #[tokio::test]
20260 async fn review_fixes_come_after_the_whole_change_summary() {
20261 let mut f = summary_fixture();
20262 f.runner.state.reviews = vec![
20263 round_with_notes(1, Some("handled the PATH case")),
20264 round_with_notes(2, None),
20265 ];
20266 let m = posted(&mut f).await;
20267 let summary = m.body.find("## Summary").expect("summary");
20268 let fixes = m.body.find("## Review rounds").expect("review rounds");
20269 assert!(summary < fixes, "{}", m.body);
20270 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
20271 assert!(m.body.contains("## Final verification"), "{}", m.body);
20272 }
20273
20274 #[tokio::test]
20275 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
20276 let mut f = summary_fixture();
20277 let first = posted(&mut f).await;
20278 let first_head = f.runner.state.change_summary.clone().unwrap().head;
20279 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
20281 let again = posted(&mut f).await;
20282 assert_eq!((again.title, again.body), (first.title, first.body));
20283
20284 let g = |args: &[&str]| {
20285 let out = std::process::Command::new("git")
20286 .args(args)
20287 .current_dir(&f.repo)
20288 .quiet()
20289 .output()
20290 .expect("spawn git");
20291 assert!(out.status.success(), "git {args:?}");
20292 };
20293 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
20294 g(&["add", "-A"]);
20295 g(&["commit", "-q", "-m", "fix(own): handle more"]);
20296 let second = posted(&mut f).await;
20297 let kept = f.runner.state.change_summary.clone().unwrap();
20298 assert_ne!(kept.head, first_head);
20299 assert!(second.body.contains("Not read yet."), "{}", second.body);
20300 assert!(!second.body.contains("so the first summary is read"));
20301 assert_eq!(second.title, "feat(own): other");
20302 }
20303
20304 #[tokio::test]
20305 async fn an_unavailable_summary_is_stated_and_never_invented() {
20306 let mut f = summary_fixture();
20307 std::fs::write(&f.reply, "no title line here").unwrap();
20308 let m = posted(&mut f).await;
20309 assert!(
20310 m.body
20311 .contains("No description of what this change does could be generated"),
20312 "{}",
20313 m.body
20314 );
20315 assert!(m.body.contains("internal reports remain in local records"));
20316 assert!(f.runner.state.change_summary.is_none());
20317 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
20318 let winner = f.runner.state.candidates[0].clone();
20320 let preview = f
20321 .runner
20322 .guarded_pr_message(&winner, None, false)
20323 .await
20324 .unwrap()
20325 .unwrap();
20326 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
20327 }
20328
20329 #[test]
20330 fn pr_message_review_single_commit_uses_its_subject() {
20331 let state = review_state(&["feat(nats): per-role user"]);
20332 let m = pr_message(&state, 'A');
20333 assert_eq!(m.title, "feat(nats): per-role user");
20334 assert!(!m.body.contains("Review the work already"), "{}", m.body);
20335 assert!(m.body.contains("## Commits under review"), "{}", m.body);
20336 }
20337
20338 #[test]
20339 fn pr_message_review_multi_commit_takes_the_oldest() {
20340 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
20341 let m = pr_message(&state, 'A');
20342 assert_eq!(m.title, "feat: the change");
20343 for s in ["feat: the change", "fix: typo", "fix: again"] {
20344 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
20345 }
20346 }
20347
20348 #[test]
20349 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
20350 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
20351 let mut state = review_state(&[first, "fix: later fixup"]);
20352 state.candidates[0].branch = "機能/ログイン".to_owned();
20353 let m = pr_message(&state, 'A');
20354 assert!(
20355 m.title.starts_with("chore: land candidate A of run"),
20356 "{}",
20357 m.title
20358 );
20359 }
20360 }
20361
20362 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
20363 BranchFacts {
20364 commits: commits
20365 .iter()
20366 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
20367 .collect(),
20368 stat: stat.to_owned(),
20369 }
20370 }
20371
20372 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
20373 let mut r = review_round(true, 0, 1, 1, true, true);
20374 r.round = round;
20375 r.fix = notes.map(|n| FixRecord {
20376 agent: "fixer".to_owned(),
20377 addressed: Vec::new(),
20378 rejected: Vec::new(),
20379 notes: n.to_owned(),
20380 committed: true,
20381 failed: None,
20382 duration_ms: 0,
20383 continuation: None,
20384 });
20385 r
20386 }
20387
20388 #[test]
20389 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
20390 let state = review_state(&["ignored"]);
20391 let f = facts(
20392 &[
20393 (
20394 "fix(login): resolve PATH on macOS",
20395 "Login shells skip rc files.",
20396 ),
20397 ("fix: address review", ""),
20398 ],
20399 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
20400 );
20401 let m = pr_message_with(&state, 'A', Some(&f));
20402 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
20403 assert!(
20404 m.body
20405 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
20406 "{}",
20407 m.body
20408 );
20409 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
20410 assert!(m.body.contains("## Diff stat"), "{}", m.body);
20411 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
20412 for banned in [
20413 "Review the work already",
20414 "no task statement",
20415 "Original task",
20416 ] {
20417 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
20418 }
20419 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
20420 }
20421
20422 #[test]
20423 fn pr_message_review_truncates_a_huge_first_commit_body() {
20424 let state = review_state(&["ignored"]);
20425 let f = facts(
20426 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
20427 "s",
20428 );
20429 let m = pr_message_with(&state, 'A', Some(&f));
20430 assert!(m.body.len() < 30_000, "{}", m.body.len());
20431 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
20432 assert!(
20433 m.body.contains("- ... 1 more commit(s)"),
20434 "{}",
20435 m.body.len()
20436 );
20437 assert!(m.body.ends_with("magi:candidate-a\n"));
20438 }
20439
20440 #[test]
20441 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
20442 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
20443 assert_eq!(m.title, "feat: x");
20444 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
20445 assert!(!m.body.contains("Diff stat"), "{}", m.body);
20446 assert!(!m.body.contains("no task statement"), "{}", m.body);
20447 }
20448
20449 #[test]
20450 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
20451 let mut state = review_state(&["日本語の件名"]);
20452 state.candidates[0].branch = "fix/macos-login-path".to_owned();
20453 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
20454 state.candidates[0].branch = "機能/ログイン".to_owned();
20455 assert!(
20456 pr_message(&state, 'A')
20457 .title
20458 .starts_with("chore: land candidate A")
20459 );
20460 }
20461
20462 #[test]
20463 fn pr_message_review_fixes_survive_a_clean_final_round() {
20464 let mut state = review_state(&["feat: x"]);
20465 state.reviews = vec![
20466 round_with_notes(1, Some("handled the PATH case")),
20467 round_with_notes(2, None),
20468 ];
20469 let body = pr_message(&state, 'A').body;
20470 assert!(
20471 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
20472 "{body}"
20473 );
20474 assert!(body.contains("### Round 1"), "{body}");
20475
20476 state.reviews = vec![
20477 round_with_notes(1, Some("first fix")),
20478 round_with_notes(2, Some("")),
20479 round_with_notes(3, Some("second fix")),
20480 round_with_notes(4, None),
20481 ];
20482 let body = pr_message(&state, 'A').body;
20483 assert!(
20484 body.contains("### Round 1") && !body.contains("first fix"),
20485 "{body}"
20486 );
20487 assert!(
20488 body.contains("### Round 3") && !body.contains("second fix"),
20489 "{body}"
20490 );
20491 assert!(body.contains("### Round 2"), "{body}");
20492 }
20493
20494 const FINAL: &str = "abcdef1234567890";
20495
20496 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20497 let mut r = round_with_notes(round, None);
20498 r.verified_head = verified.map(str::to_owned);
20499 r
20500 }
20501
20502 fn ok_gate(state: &mut RunState, head: Option<&str>) {
20503 state.gate = vec![CommandOutcome {
20504 command: "gate".to_owned(),
20505 code: Some(0),
20506 output_tail: String::new(),
20507 duration_ms: 0,
20508 resource_blocked: false,
20509 }];
20510 state.gate_ran = true;
20511 state.gate_head = head.map(str::to_owned);
20512 }
20513
20514 fn body_at(state: &RunState) -> String {
20515 pr_message_raw(
20516 state,
20517 'A',
20518 None,
20519 crate::pr_outcome::FinalHead::At(FINAL),
20520 ChangeNote::NotAsked,
20521 )
20522 .body
20523 }
20524
20525 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20526
20527 #[test]
20528 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20529 let mut state = review_state(&["feat: x"]);
20530 state.reviews = vec![
20531 round_with_notes(1, Some(SANDBOXED)),
20532 passing_round(2, Some(FINAL)),
20533 ];
20534 ok_gate(&mut state, Some(FINAL));
20535 let body = body_at(&state);
20536 assert!(!body.contains("Handled the PATH case."), "{body}");
20537 assert!(!body.contains("could not commit"), "{body}");
20538 assert!(!body.contains("could not run"), "{body}");
20539 assert!(
20540 body.contains("The fix is committed on the branch"),
20541 "{body}"
20542 );
20543 assert!(!body.contains("sandbox"), "{body}");
20544 assert!(
20545 body.contains("Tests (`verify.e2e`): passed on the final commit"),
20546 "{body}"
20547 );
20548 assert!(
20549 body.contains("Gate (`verify.gate`): passed on the final commit"),
20550 "{body}"
20551 );
20552 assert!(
20554 state.reviews[0]
20555 .fix
20556 .as_ref()
20557 .unwrap()
20558 .notes
20559 .contains("could not commit")
20560 );
20561 }
20562
20563 #[test]
20564 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20565 let mut state = review_state(&["feat: x"]);
20566 let mut r1 = round_with_notes(1, Some(SANDBOXED));
20567 r1.fix.as_mut().unwrap().committed = false;
20568 let mut r2 = passing_round(2, Some(FINAL));
20569 r2.e2e[0].code = Some(1);
20570 state.reviews = vec![r1, r2];
20571 let body = body_at(&state);
20572 assert!(!body.contains("could not commit"), "{body}");
20573 assert!(!body.contains("could not run"), "{body}");
20574 assert!(body.contains("This round produced no commit"), "{body}");
20575 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20576 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20577 assert!(!body.contains("passed"), "{body}");
20578 }
20579
20580 #[test]
20581 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20582 let mut state = review_state(&["feat: x"]);
20583 state.reviews = vec![
20584 round_with_notes(1, Some(SANDBOXED)),
20585 passing_round(2, Some("0123456789abcdef")),
20586 ];
20587 ok_gate(&mut state, Some("0123456789abcdef"));
20588 let body = body_at(&state);
20589 assert!(!body.contains("HTTP tests could not run"), "{body}");
20590 assert!(!body.contains("could not commit"), "{body}");
20592 assert!(
20593 body.contains("passed on an earlier commit (`0123456`)"),
20594 "{body}"
20595 );
20596 assert!(!body.contains("passed on the final commit"), "{body}");
20597
20598 ok_gate(&mut state, None);
20600 let body = body_at(&state);
20601 assert!(
20602 body.contains("passed on a commit that was not recorded"),
20603 "{body}"
20604 );
20605 }
20606
20607 #[test]
20608 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20609 let state = state_with_summary(
20610 "add retries to the client",
20611 "TITLE: feat: retries\n\nDid it.",
20612 );
20613 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20614 assert_eq!(m.title, "feat: retries");
20615 assert!(
20616 m.body.contains("<summary>Original task</summary>"),
20617 "{}",
20618 m.body
20619 );
20620 assert!(!m.body.contains("Commits under review"), "{}", m.body);
20621 assert!(
20622 m.body
20623 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20624 "{}",
20625 m.body
20626 );
20627 }
20628
20629 #[test]
20630 fn pr_message_review_bounds_a_long_english_subject() {
20631 let long = format!("feat: {}", "word ".repeat(100));
20632 let m = pr_message(&review_state(&[&long]), 'A');
20633 assert!(m.title.starts_with("feat: word"), "{}", m.title);
20634 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20635 }
20636
20637 #[test]
20638 fn pr_message_implementation_run_is_unchanged_by_review_support() {
20639 let state = state_with_summary("add retries\n\ndetails", "- did some things");
20640 let m = pr_message(&state, 'A');
20641 assert_eq!(m.title, "add retries");
20642 assert!(m.body.contains("<summary>Original task</summary>"));
20643 assert!(!m.body.contains("Commits under review"));
20644 assert_eq!(landing_subject_source(&state), state.instruction);
20645 }
20646
20647 #[test]
20648 fn review_run_squash_subject_is_the_change_not_the_prompt() {
20649 let state = review_state(&["feat: the change", "fix: typo"]);
20650 let source = landing_subject_source(&state);
20651 assert_eq!(land::merge_subject("", &source), "feat: the change");
20652 assert_eq!(
20653 land::merge_subject("magi: candidate A (uncommitted work)", &source),
20654 "feat: the change"
20655 );
20656 assert_eq!(
20658 land::merge_subject("feat: renamed by hand", &source),
20659 "feat: renamed by hand"
20660 );
20661 let blank = review_state(&["日本語"]);
20662 assert!(
20663 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20664 );
20665 }
20666
20667 #[test]
20668 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20669 let state = review_state(&["feat: the change"]);
20670 let source = landing_subject_source(&state);
20671 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20672 assert_eq!(
20673 land::merge_subject(landing_title(&state, old), &source),
20674 "feat: the change"
20675 );
20676 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20677 let task = state_with_summary("add retries", "");
20678 assert_eq!(landing_title(&task, old), old);
20679 }
20680
20681 #[test]
20682 fn pr_message_describes_the_change_not_the_task() {
20683 let state = state_with_summary(
20684 "今回やってほしいこと: results projector を直す",
20685 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20686 );
20687 let m = pr_message(&state, 'A');
20688 assert_eq!(m.title, "fix(web): batch the runs list reads");
20689 assert!(
20690 m.body.starts_with("## Summary\n\n- reads run.json once"),
20691 "{}",
20692 m.body
20693 );
20694 assert!(!m.body.contains("TITLE:"), "{}", m.body);
20695 let task_at = m.body.find("今回やってほしいこと").unwrap();
20696 let details_at = m.body.find("<details>").unwrap();
20697 assert!(
20698 details_at < task_at,
20699 "the task lives inside <details>: {}",
20700 m.body
20701 );
20702 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20703 assert!(m.body.contains("magi:candidate-a"));
20704 }
20705
20706 #[test]
20707 fn pr_message_falls_back_to_the_task_without_a_title_line() {
20708 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20709 let m = pr_message(&state, 'A');
20710 assert_eq!(m.title, "add retries");
20711 assert!(
20712 m.body.contains("## Summary\n\n- did some things"),
20713 "{}",
20714 m.body
20715 );
20716
20717 let none = RunState::new(
20718 PathBuf::from("/repo"),
20719 "main".to_owned(),
20720 "abc1234".to_owned(),
20721 "add retries".to_owned(),
20722 Config::default(),
20723 );
20724 let m = pr_message(&none, 'A');
20725 assert_eq!(m.title, "add retries");
20726 assert!(!m.body.contains("## Summary"), "{}", m.body);
20727 }
20728
20729 #[test]
20730 fn pr_message_refuses_the_candidate_commit_subject() {
20731 for bad in [
20732 "TITLE: magi: candidate A (uncommitted work)",
20733 "TITLE: chore: stuff (uncommitted work)",
20734 "TITLE: ",
20735 ] {
20736 let state = state_with_summary("add retries", bad);
20737 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20738 }
20739 }
20740
20741 #[test]
20742 fn pr_message_bounds_a_very_long_task_and_title() {
20743 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20744 let state = state_with_summary(&long, "- nothing");
20745 let m = pr_message(&state, 'A');
20746 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20747 assert!(!m.title.contains('\n'));
20748
20749 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20750 let m = pr_message(&state, 'A');
20751 assert!(m.title.starts_with("feat: "));
20752 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20753 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20754 }
20755
20756 fn long_title_of(instruction: &str) -> String {
20757 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20758 }
20759
20760 #[test]
20761 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20762 let first = "Make the landing path keep a readable title for long tasks";
20763 let line = format!(
20764 "{first}. {}",
20765 "And then keep going with more words ".repeat(20)
20766 );
20767 let t = long_title_of(&line);
20768 assert_eq!(t, first);
20769 assert!(!t.starts_with("chore: land"));
20770 }
20771
20772 #[test]
20773 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20774 let line = "word ".repeat(200);
20775 let t = long_title_of(&line);
20776 assert!(t.ends_with("word..."), "{t}");
20777 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20778 }
20779
20780 #[test]
20781 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20782 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20783 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20784 }
20785 }
20786
20787 #[test]
20788 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20789 let mut state = state_with_summary("再試行を追加する", "- nothing");
20790 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20791 state.config.graph.github_language = "ja".into();
20792 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20793 }
20794
20795 #[test]
20796 fn pr_message_title_limit_is_exact() {
20797 let at = "a".repeat(PR_TITLE_MAX);
20798 assert_eq!(long_title_of(&at), at);
20799 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20800 assert!(over.ends_with("..."), "{over}");
20801 assert_eq!(over.chars().count(), PR_TITLE_MAX);
20802 }
20803
20804 #[test]
20805 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20806 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20807 let t = long_title_of(&line);
20808 assert!(t.ends_with("..."), "{t}");
20809 assert!(t.is_ascii(), "{t}");
20810 }
20811
20812 #[test]
20813 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20814 let line = format!(
20815 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20816 "plus more filler words ".repeat(20)
20817 );
20818 let t = long_title_of(&line);
20819 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20820 }
20821
20822 #[test]
20823 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20824 let line = format!("\n\n# {}", "title words ".repeat(40));
20825 let state = state_with_summary(&line, "- nothing");
20826 let m = pr_message(&state, 'A');
20827 assert!(m.title.starts_with("title words"), "{}", m.title);
20828 assert_eq!(
20829 land::merge_subject(&m.title, &landing_subject_source(&state)),
20830 m.title
20831 );
20832 assert_eq!(
20834 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20835 "feat: renamed by hand"
20836 );
20837 }
20838
20839 #[tokio::test]
20840 async fn github_text_rewrite_is_bounded_and_falls_back() {
20841 crate::run::pin_test_home();
20842 for (reply, accepted) in [
20843 (
20844 "TITLE: fix: retries\nAdd retries for failed requests.",
20845 true,
20846 ),
20847 (
20848 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20849 false,
20850 ),
20851 (
20852 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20853 true,
20854 ),
20855 (
20856 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20857 true,
20858 ),
20859 (
20860 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20861 true,
20862 ),
20863 (
20864 "TITLE: fix: retries\n日本語の説明です。 token=secret",
20865 false,
20866 ),
20867 ("TITLE: fix: retries\n", false),
20868 ] {
20869 let dir = tempfile::tempdir().unwrap();
20870 let mut runner = runner_at(RunStatus::Gating);
20871 runner.state = state_with_summary(
20872 "add retries",
20873 "TITLE: fix: retries\n日本語の説明を書きます。",
20874 );
20875 runner.state.repo = dir.path().to_owned();
20876 runner.state.candidates[0].worktree = dir.path().to_owned();
20877 let mut author = spec("alpha");
20878 author.command = vec![
20879 "sh".into(),
20880 "-c".into(),
20881 "printf '%s' \"$REWRITE_REPLY\"".into(),
20882 ];
20883 author.env.insert("REWRITE_REPLY".into(), reply.into());
20884 runner.state.config.agents = vec![author];
20885 seed_public_summary(&mut runner);
20886 let winner = runner.state.candidates[0].clone();
20887 let first = runner
20888 .guarded_pr_message(&winner, None, true)
20889 .await
20890 .unwrap();
20891 let message = if accepted {
20892 first.expect("a passing rewrite never asks")
20893 } else {
20894 assert!(first.is_none());
20896 assert!(runner.state.parked);
20897 let store = ask::Questions::open();
20898 let asked: Vec<_> = store
20899 .list()
20900 .into_iter()
20901 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20902 .collect();
20903 assert_eq!(asked.len(), 1);
20904 assert!(!asked[0].detail.contains("secret"));
20905 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20906 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20907 assert_eq!(
20908 asked[0]
20909 .choices
20910 .contains(&crate::github_text::USE_REWRITE.to_owned()),
20911 rewrite.is_some()
20912 );
20913 store
20916 .update(&asked[0].id, |q| {
20917 q.abandon("test");
20918 Ok(())
20919 })
20920 .unwrap();
20921 let again = runner
20922 .guarded_pr_message(&winner, None, true)
20923 .await
20924 .unwrap()
20925 .expect("an abandoned question falls back");
20926 let n = store
20927 .list()
20928 .into_iter()
20929 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20930 .count();
20931 assert_eq!(n, 1, "asked once per rejected text");
20932 again
20933 };
20934 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20935 assert!(!message.body.contains("secret"));
20936 assert!(!message.body.contains("prefix"));
20937 assert!(!message.title.contains("secret"));
20938 assert_eq!(
20939 message.body.contains("Add retries for failed requests."),
20940 accepted
20941 );
20942 assert_eq!(
20943 runner.state.seats["github-text-rewrite"].turns, 1,
20944 "only one rewrite invocation"
20945 );
20946 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20947 if accepted {
20948 assert!(runner.state.github_text.is_none());
20949 assert!(!runner.state.parked);
20950 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20951 assert!(message.body.contains("## Final verification"));
20952 assert!(message.body.contains("Evidence: final diff"));
20953 assert!(
20954 message
20955 .body
20956 .contains(&format!("magi:run/{}", runner.state.id))
20957 );
20958 }
20959 if !accepted {
20960 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20961 assert!(
20962 message
20963 .body
20964 .contains(&format!("magi:run/{}", runner.state.id))
20965 );
20966 }
20967 }
20968 }
20969
20970 fn scripted(id: &str, reply: &str) -> AgentSpec {
20971 let mut a = spec(id);
20972 a.command = vec![
20973 "sh".into(),
20974 "-c".into(),
20975 "printf '%s' \"$REWRITE_REPLY\"".into(),
20976 ];
20977 a.env.insert("REWRITE_REPLY".into(), reply.into());
20978 a
20979 }
20980
20981 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20982 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20983
20984 fn seed_public_summary(runner: &mut Runner) {
20986 let repo = &runner.state.repo;
20987 let g = |args: &[&str]| {
20988 let out = std::process::Command::new("git")
20989 .args(args)
20990 .current_dir(repo)
20991 .quiet()
20992 .output()
20993 .unwrap();
20994 assert!(out.status.success(), "git {args:?}");
20995 String::from_utf8_lossy(&out.stdout).trim().to_owned()
20996 };
20997 g(&["init", "-q", "-b", "main"]);
20998 g(&[
20999 "-c",
21000 "user.name=Test",
21001 "-c",
21002 "user.email=test@example.invalid",
21003 "commit",
21004 "-q",
21005 "--allow-empty",
21006 "-m",
21007 "feat: fixture",
21008 ]);
21009 let head = g(&["rev-parse", "HEAD"]);
21010 g(&["update-ref", "refs/remotes/origin/main", &head]);
21011 let evidence = crate::pr_outcome::summary_evidence(
21012 &runner.state,
21013 crate::pr_outcome::FinalHead::At(&head),
21014 );
21015 let generated = &runner.state.candidates[0].summary;
21016 runner.state.change_summary = Some(crate::run::ChangeSummary {
21017 base: head.clone(),
21018 head,
21019 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
21020 title: summary_title(generated).unwrap(),
21021 body: summary_without_title(generated),
21022 });
21023 }
21024
21025 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
21026 let mut runner = runner_at(RunStatus::Gating);
21027 runner.state = state_with_summary("add retries", JA_SUMMARY);
21028 if review_only {
21029 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21030 }
21031 runner.state.repo = dir.to_owned();
21032 runner.state.candidates[0].worktree = dir.to_owned();
21033 runner.state.config.agents = agents;
21034 seed_public_summary(&mut runner);
21035 runner
21036 }
21037
21038 fn gate_questions(runner: &Runner) -> usize {
21039 ask::Questions::open()
21040 .list()
21041 .into_iter()
21042 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
21043 .count()
21044 }
21045
21046 #[tokio::test]
21047 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
21048 crate::run::pin_test_home();
21049 let dir = tempfile::tempdir().unwrap();
21050 let mut runner = rewrite_runner(
21051 dir.path(),
21052 true,
21053 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
21054 );
21055 runner.state.config.roles.language_judge =
21056 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
21057 let winner = runner.state.candidates[0].clone();
21058 let msg = runner
21059 .guarded_pr_message(&winner, None, true)
21060 .await
21061 .unwrap()
21062 .expect("an English rewrite posts without asking");
21063 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
21064 assert!(msg.body.contains("<details>kept</details>"));
21065 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
21066 assert_eq!(gate_questions(&runner), 0);
21067 assert!(runner.state.github_text.is_none());
21068 assert!(!runner.state.seats.contains_key("impl-A"));
21069 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21070 }
21071
21072 #[tokio::test]
21073 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
21074 crate::run::pin_test_home();
21075 let dir = tempfile::tempdir().unwrap();
21076 let mut runner = rewrite_runner(
21077 dir.path(),
21078 false,
21079 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
21080 );
21081 runner.state.config.roles.language_judge =
21082 Some(crate::config::AgentChoice::One("beta".into()));
21083 runner.state.seats.insert(
21084 "impl-A".into(),
21085 SeatState::new("impl-A", "alpha", runner.state.seed),
21086 );
21087 let winner = runner.state.candidates[0].clone();
21088 let msg = runner
21089 .guarded_pr_message(&winner, None, true)
21090 .await
21091 .unwrap()
21092 .expect("rewritten");
21093 assert!(msg.body.contains("<details>kept</details>"));
21094 assert_eq!(runner.state.seats["impl-A"].turns, 0);
21095 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21096 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
21097 }
21098
21099 #[tokio::test]
21100 async fn rewrite_reuses_a_resumable_implementer_session() {
21101 crate::run::pin_test_home();
21102 let dir = tempfile::tempdir().unwrap();
21103 let mut runner = rewrite_runner(
21104 dir.path(),
21105 false,
21106 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
21107 );
21108 runner.state.config.roles.language_judge =
21109 Some(crate::config::AgentChoice::One("beta".into()));
21110 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
21111 seat.turns = 1;
21112 runner.state.seats.insert("impl-A".into(), seat);
21113 let winner = runner.state.candidates[0].clone();
21114 runner
21115 .guarded_pr_message(&winner, None, true)
21116 .await
21117 .unwrap()
21118 .expect("rewritten");
21119 assert_eq!(runner.state.seats["impl-A"].turns, 2);
21120 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
21121 }
21122
21123 #[tokio::test]
21124 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
21125 crate::run::pin_test_home();
21128 let dir = tempfile::tempdir().unwrap();
21129 let mut exhausted = spec("alpha");
21130 exhausted.command = vec!["false".into()];
21131 let mut runner = rewrite_runner(
21132 dir.path(),
21133 false,
21134 vec![exhausted, scripted("beta", EN_REWRITE)],
21135 );
21136 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
21138 "alpha".into(),
21139 "beta".into(),
21140 ]));
21141 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
21142 seat.turns = 1;
21143 runner.state.seats.insert("impl-A".into(), seat);
21144 let winner = runner.state.candidates[0].clone();
21145 let msg = runner
21146 .guarded_pr_message(&winner, None, true)
21147 .await
21148 .unwrap()
21149 .expect("the role rewrites after the implementer fails");
21150 assert!(msg.body.contains("`src/net.rs`"));
21151 assert_eq!(
21152 runner.state.seats["impl-A"].turns, 2,
21153 "the implementer was tried once"
21154 );
21155 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
21156 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21157 }
21158
21159 #[tokio::test]
21160 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
21161 crate::run::pin_test_home();
21162 let dir = tempfile::tempdir().unwrap();
21163 let mut broken = spec("bad");
21164 broken.command = vec!["false".into()];
21165 let mut runner = rewrite_runner(
21166 dir.path(),
21167 true,
21168 vec![
21169 broken,
21170 scripted("noformat", "Add retries."),
21171 scripted("good", EN_REWRITE),
21172 ],
21173 );
21174 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
21175 "bad".into(),
21176 "noformat".into(),
21177 "good".into(),
21178 ]));
21179 let winner = runner.state.candidates[0].clone();
21180 let msg = runner
21181 .guarded_pr_message(&winner, None, true)
21182 .await
21183 .unwrap()
21184 .expect("the third agent rewrites");
21185 assert!(msg.body.contains("`src/net.rs`"));
21186 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
21187 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
21188 }
21189
21190 #[tokio::test]
21191 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
21192 crate::run::pin_test_home();
21193 for (agents, role, saved) in [
21195 (
21196 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
21197 Some("beta"),
21198 true,
21199 ),
21200 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
21201 (Vec::new(), None, false),
21202 ] {
21203 let dir = tempfile::tempdir().unwrap();
21204 let mut runner = rewrite_runner(dir.path(), true, agents);
21205 runner.state.config.roles.language_judge =
21206 role.map(|r| crate::config::AgentChoice::One(r.into()));
21207 let winner = runner.state.candidates[0].clone();
21208 let first = runner
21209 .guarded_pr_message(&winner, None, true)
21210 .await
21211 .unwrap();
21212 assert!(first.is_none(), "withheld text parks the run");
21213 assert!(runner.state.parked);
21214 assert_eq!(gate_questions(&runner), 1);
21215 let g = runner.state.github_text.as_ref().unwrap();
21216 assert_eq!(g.rewrite.is_some(), saved);
21217 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
21218 }
21219 }
21220
21221 #[tokio::test]
21222 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
21223 crate::run::pin_test_home();
21224 let dir = tempfile::tempdir().unwrap();
21225 let mut runner = runner_at(RunStatus::Gating);
21226 runner.state =
21227 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
21228 runner.state.repo = dir.path().to_owned();
21229 runner.state.candidates[0].worktree = dir.path().to_owned();
21230 let mut author = spec("alpha");
21231 author.command = vec![
21232 "sh".into(),
21233 "-c".into(),
21234 "printf '%s' \"$REWRITE_REPLY\"".into(),
21235 ];
21236 author.env.insert(
21237 "REWRITE_REPLY".into(),
21238 format!(
21239 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
21240 "English sentence. ".repeat(600)
21241 ),
21242 );
21243 runner.state.config.agents = vec![author];
21244 seed_public_summary(&mut runner);
21245 let winner = runner.state.candidates[0].clone();
21246 assert!(
21247 runner
21248 .guarded_pr_message(&winner, None, true)
21249 .await
21250 .unwrap()
21251 .is_none()
21252 );
21253 let record = runner.state.github_text.clone().unwrap();
21254 let rewrite = record.rewrite.clone().unwrap();
21255 assert_eq!(rewrite.failed, ["body-language"]);
21256 assert!(
21257 rewrite
21258 .body
21259 .ends_with(&format!("magi:run/{}", runner.state.id))
21260 );
21261 let store = ask::Questions::open();
21262 let q = store.get(record.question.as_ref().unwrap()).unwrap();
21263 assert!(
21264 q.choices
21265 .contains(&crate::github_text::USE_REWRITE.to_owned())
21266 );
21267 for label in [
21268 "ORIGINAL",
21269 "FALLBACK",
21270 "REWRITE",
21271 "Rules still failed: body-language",
21272 "truncated",
21273 ] {
21274 assert!(q.detail.contains(label), "{label}");
21275 }
21276 let artifact = std::fs::read_to_string(crate::run::artifact_path(
21277 &runner.state,
21278 &crate::github_text::artifact_name(&record.fingerprint),
21279 ))
21280 .unwrap();
21281 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
21282 runner.state =
21284 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
21285 store
21286 .update(&q.id, |q| {
21287 q.say("まだ日本語")?;
21288 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
21289 })
21290 .unwrap();
21291 assert!(
21292 runner
21293 .guarded_pr_message(&winner, None, true)
21294 .await
21295 .unwrap()
21296 .is_none()
21297 );
21298 let retry = runner.state.github_text.as_ref().unwrap();
21299 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
21300 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
21301 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
21302 assert!(
21303 q.posting_comparison
21304 .as_ref()
21305 .unwrap()
21306 .matches_detail(&q.detail)
21307 );
21308 store
21309 .update(&q.id, |q| {
21310 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21311 })
21312 .unwrap();
21313 let message = runner
21314 .guarded_pr_message(&winner, None, true)
21315 .await
21316 .unwrap()
21317 .unwrap();
21318 assert_eq!(message.title, rewrite.title);
21319 assert_eq!(message.body, rewrite.body);
21320 assert_eq!(
21321 runner.state.github_text.as_ref().unwrap().choice,
21322 Some(GithubTextChoice::Rewrite)
21323 );
21324 let again = runner
21325 .guarded_pr_message(&winner, None, true)
21326 .await
21327 .unwrap()
21328 .unwrap();
21329 assert_eq!(again.title, message.title);
21330 assert_eq!(again.body, message.body);
21331 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21332 assert!(
21333 !runner
21334 .state
21335 .events
21336 .iter()
21337 .any(|e| e.message.contains(&rewrite.title)
21338 || e.message.contains("English sentence."))
21339 );
21340 }
21341
21342 #[tokio::test]
21343 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
21344 crate::run::pin_test_home();
21345 let dir = tempfile::tempdir().unwrap();
21346 let mut runner = runner_at(RunStatus::Gating);
21347 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
21348 runner.state.repo = dir.path().to_owned();
21349 runner.state.candidates[0].worktree = dir.path().to_owned();
21350 seed_public_summary(&mut runner);
21351 let winner = runner.state.candidates[0].clone();
21352 assert!(
21353 runner
21354 .guarded_pr_message(&winner, None, true)
21355 .await
21356 .unwrap()
21357 .is_none()
21358 );
21359 let record = runner.state.github_text.as_ref().unwrap();
21360 assert!(record.rewrite.is_none());
21361 let mut old = serde_json::to_value(record).unwrap();
21363 old.as_object_mut().unwrap().remove("rewrite");
21364 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
21365 assert!(old.rewrite.is_none());
21366 let id = record.question.clone().unwrap();
21367 ask::Questions::open()
21368 .update(&id, |q| {
21369 q.choices.push(crate::github_text::USE_REWRITE.into());
21370 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21371 })
21372 .unwrap();
21373 let message = runner
21374 .guarded_pr_message(&winner, None, true)
21375 .await
21376 .unwrap()
21377 .unwrap();
21378 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
21379 assert_eq!(
21380 runner.state.github_text.as_ref().unwrap().choice,
21381 Some(GithubTextChoice::Fallback)
21382 );
21383 }
21384
21385 #[tokio::test]
21386 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
21387 crate::run::pin_test_home();
21388 let dir = tempfile::tempdir().unwrap();
21389 let mut runner = runner_at(RunStatus::Gating);
21390 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
21391 runner.state.repo = dir.path().to_owned();
21392 runner.state.candidates[0].worktree = dir.path().to_owned();
21393 seed_public_summary(&mut runner);
21394 let winner = runner.state.candidates[0].clone();
21395 assert!(
21396 runner
21397 .guarded_pr_message(&winner, None, true)
21398 .await
21399 .unwrap()
21400 .is_none()
21401 );
21402 let store = ask::Questions::open();
21403 let find = |run: &str| {
21404 store
21405 .list()
21406 .into_iter()
21407 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
21408 .unwrap()
21409 };
21410 let run = runner.state.id.clone();
21411 let q = find(&run);
21412 let fp = runner
21413 .state
21414 .github_text
21415 .as_ref()
21416 .unwrap()
21417 .fingerprint
21418 .clone();
21419 let name = crate::github_text::artifact_name(&fp);
21420 let path = crate::run::artifact_path(&runner.state, &name);
21421 assert!(q.detail.contains(&path.display().to_string()));
21422 assert!(q.detail.contains("title-language: title"));
21423 let saved = std::fs::read_to_string(&path).expect("artifact written");
21424 assert!(saved.contains("日本語のタイトル"));
21425 assert!(q.cwd.is_none());
21426 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
21427 let say = |id: &str, text: &str, choice: &str| {
21429 store
21430 .update(id, |q| {
21431 q.thread.push(ask::Turn {
21432 who: ask::Who::Operator,
21433 body: text.to_owned(),
21434 at: jiff::Timestamp::now(),
21435 note: None,
21436 });
21437 q.answer(ask::Answer::Choice(choice.to_owned()))
21438 })
21439 .unwrap();
21440 };
21441 say(&q.id, "まだ日本語", "use my text");
21442 assert!(
21443 runner
21444 .guarded_pr_message(&winner, None, true)
21445 .await
21446 .unwrap()
21447 .is_none()
21448 );
21449 let second = find(&run);
21450 assert_ne!(second.id, q.id);
21451 assert!(second.detail.contains("did not pass"));
21452 assert!(second.detail.contains("日本語のタイトル"));
21455 assert!(!second.detail.contains("まだ日本語"));
21456 assert!(second.detail.contains(&path.display().to_string()));
21457 say(&second.id, "fix: retry failed requests", "use my text");
21458 let message = runner
21459 .guarded_pr_message(&winner, None, true)
21460 .await
21461 .unwrap()
21462 .expect("a vetted replacement is posted");
21463 assert_eq!(message.title, "fix: retry failed requests");
21464 assert_eq!(
21465 runner
21466 .state
21467 .github_text
21468 .as_ref()
21469 .unwrap()
21470 .chosen_title
21471 .as_deref(),
21472 Some("fix: retry failed requests")
21473 );
21474 assert!(runner.state.github_text.as_ref().unwrap().resolved);
21475 let again = runner
21477 .guarded_pr_message(&winner, None, true)
21478 .await
21479 .unwrap()
21480 .unwrap();
21481 assert_eq!(again.title, message.title);
21482 assert_eq!(
21483 store
21484 .list()
21485 .into_iter()
21486 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
21487 .count(),
21488 2
21489 );
21490 }
21491
21492 #[test]
21493 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21494 let mut state = state_with_summary(
21498 "add retries",
21499 "TITLE: fix(web): batch reads\n- reads run.json once",
21500 );
21501 state.config.graph.language = "ja".to_owned();
21502 let m = pr_message(&state, 'A');
21503 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21504 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21505
21506 let task = "今回やってほしいこと: results projector を直す";
21509 let mut state = state_with_summary(task, "- no title line");
21510 state.config.graph.language = "ja".to_owned();
21511 let m = pr_message(&state, 'A');
21512 assert_eq!(
21513 m.title,
21514 format!("chore: land candidate A of run {}", state.id)
21515 );
21516 assert!(
21517 m.body.contains(&format!(
21518 "<summary>Original task</summary>\n\n{task}\n\n</details>"
21519 )),
21520 "{}",
21521 m.body
21522 );
21523 }
21524
21525 #[test]
21526 fn pr_message_scrubs_home_paths_and_addresses() {
21527 let state = state_with_summary(
21528 "fix it in /Users/someone/src/x",
21529 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21530 );
21531 let m = pr_message(&state, 'A');
21532 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21533 assert!(!m.body.contains(leak), "{}", m.body);
21534 }
21535 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21536 }
21537
21538 #[test]
21539 fn pr_message_survives_a_task_that_closes_details() {
21540 let state = state_with_summary("a </details> b", "TITLE: fix: x");
21541 let m = pr_message(&state, 'A');
21542 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21543 }
21544
21545 #[test]
21546 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21547 let cmd = manual_merge_command(
21548 MergeStyle::Squash,
21549 Path::new("/repo"),
21550 "b",
21551 "fix: \"quoted\" $(x) `y`\n\nbody",
21552 );
21553 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21554 }
21555
21556 #[test]
21557 fn manual_merge_command_matches_the_configured_style() {
21558 let repo = Path::new("/repo");
21559 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21560
21561 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21562 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21563
21564 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21565 assert_eq!(
21566 squash,
21567 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21568 \"Merge magi run 0832 (candidate A)\""
21569 );
21570
21571 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21572 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21573 }
21574
21575 #[test]
21576 fn a_nudge_gets_a_quarter_of_the_budget() {
21577 assert_eq!(retry_budget(secs(1200), true), secs(300));
21579 assert_eq!(retry_budget(secs(3600), true), secs(900));
21580 }
21581
21582 #[test]
21583 fn a_resent_prompt_keeps_the_whole_budget() {
21584 assert_eq!(retry_budget(secs(1200), false), secs(1200));
21587 assert_eq!(retry_budget(secs(60), false), secs(60));
21588 }
21589
21590 #[test]
21591 fn the_floor_never_exceeds_the_original_budget() {
21592 assert_eq!(retry_budget(secs(60), true), secs(60));
21596 assert_eq!(retry_budget(secs(480), true), secs(120));
21597 assert_eq!(retry_budget(secs(0), true), secs(0));
21598 }
21599
21600 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21601 agent::CommandEvidence {
21602 id: "item1".to_owned(),
21603 description: "cargo test".to_owned(),
21604 exit_code,
21605 result_summary: String::new(),
21606 source: "codex".to_owned(),
21607 }
21608 }
21609
21610 #[test]
21611 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21612 assert!(!has_unconfirmed_command(&[]));
21616 }
21617
21618 #[test]
21619 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21620 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21624 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21625 assert!(!has_unconfirmed_command(&[
21626 evidence(Some(0)),
21627 evidence(Some(101))
21628 ]));
21629 }
21630
21631 #[test]
21632 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21633 assert!(has_unconfirmed_command(&[
21634 evidence(Some(0)),
21635 evidence(None)
21636 ]));
21637 }
21638
21639 #[test]
21640 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21641 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21642 assert_eq!(
21643 verified_noop_claim(true, &[], text).as_deref(),
21644 Some("already fixed by b32cfc4, on main.")
21645 );
21646 }
21647
21648 #[test]
21649 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21650 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21653 assert!(verified_noop_claim(false, &[], text).is_none());
21654 }
21655
21656 #[test]
21657 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21658 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21659 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21660 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21662 }
21663
21664 #[test]
21665 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21666 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21667 }
21668
21669 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21672 runner.state.candidates = shape
21673 .iter()
21674 .enumerate()
21675 .map(|(i, &(empty, verified))| Candidate {
21676 index: i,
21677 label: (b'A' + i as u8) as char,
21678 agent: "sonnet".to_owned(),
21679 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21680 worktree: PathBuf::from(format!("/wt/{i}")),
21681 summary: String::new(),
21682 stat: String::new(),
21683 files: 0,
21684 commits: 0,
21685 empty,
21686 failed: None,
21687 verified_noop: verified.map(str::to_owned),
21688 duration_ms: 0,
21689 folded: false,
21690 })
21691 .collect();
21692 }
21693
21694 #[test]
21695 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21696 ask_test_home();
21697 let mut runner = runner_at(RunStatus::Implementing);
21698 set_candidates(
21699 &mut runner,
21700 &[
21701 (true, Some("already on main at b32cfc4")),
21702 (true, Some("same fix, see the existing test")),
21703 ],
21704 );
21705
21706 runner
21707 .after_implement()
21708 .expect("a verified no-op is not an error");
21709
21710 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21711 }
21712
21713 #[test]
21714 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21715 ask_test_home();
21716 let mut runner = runner_at(RunStatus::Implementing);
21717 set_candidates(
21721 &mut runner,
21722 &[(true, Some("already on main at b32cfc4")), (true, None)],
21723 );
21724
21725 let err = runner
21726 .after_implement()
21727 .expect_err("an unverified empty candidate must still fail the run");
21728
21729 assert!(
21730 err.to_string().contains("no candidate produced a change"),
21731 "{err}"
21732 );
21733 assert_eq!(runner.state.status, RunStatus::Failed);
21734 }
21735
21736 #[test]
21737 fn after_implement_still_fails_an_ordinary_all_empty_run() {
21738 ask_test_home();
21739 let mut runner = runner_at(RunStatus::Implementing);
21740 set_candidates(&mut runner, &[(true, None), (true, None)]);
21741
21742 let err = runner
21743 .after_implement()
21744 .expect_err("no candidate declared anything; this is an ordinary failure");
21745
21746 assert!(
21747 err.to_string().contains("no candidate produced a change"),
21748 "{err}"
21749 );
21750 assert_eq!(runner.state.status, RunStatus::Failed);
21751 }
21752}
21753
21754#[derive(Debug)]
21757struct GraftRestoreFailed(String);
21758
21759impl std::fmt::Display for GraftRestoreFailed {
21760 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21761 write!(
21762 f,
21763 "could not restore the winner to its pre-graft commit: {}",
21764 self.0
21765 )
21766 }
21767}
21768
21769impl std::error::Error for GraftRestoreFailed {}
21770
21771#[cfg(test)]
21775mod resource_wait_tests {
21776 use super::*;
21777 use std::collections::BTreeMap;
21778
21779 fn sh(id: &str, script: &str) -> AgentSpec {
21780 AgentSpec {
21781 id: id.to_owned(),
21782 kind: crate::config::AgentKind::Command,
21783 model: None,
21784 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21785 extra_args: Vec::new(),
21786 env: BTreeMap::new(),
21787 prompt_delivery: None,
21788 max_turn_secs: None,
21789 }
21790 }
21791
21792 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21793 SeatJob {
21794 seat: SeatState::new("impl-A", &spec.id, 1),
21795 spec,
21796 cwd: dir.to_path_buf(),
21797 prompt: "do it".to_owned(),
21798 timeout,
21799 allow_write: true,
21800 sessions: false,
21801 artifacts: dir.join("art"),
21802 stem: "impl-A".to_owned(),
21803 handover: None,
21804 }
21805 }
21806
21807 fn state(resource_wait_secs: u64) -> RunState {
21808 let mut config = Config::default();
21809 config.graph.timeout_resource_wait = resource_wait_secs;
21810 RunState::new(
21811 PathBuf::from("/repo"),
21812 "main".to_owned(),
21813 "abc1234".to_owned(),
21814 "task".to_owned(),
21815 config,
21816 )
21817 }
21818
21819 fn hold(cache: &Path) -> crate::cache::Guard {
21820 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21821 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21822 crate::cache::AcquireOutcome::Acquired(g) => g,
21823 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21824 }
21825 }
21826
21827 async fn run(
21828 jobs: Vec<SeatJob>,
21829 sem: Arc<Semaphore>,
21830 cache: Option<&Path>,
21831 st: &mut RunState,
21832 ) -> Vec<(usize, SeatState, AgentOutcome)> {
21833 let prompts = Prompts::default();
21834 let ctx = WaveCtx {
21835 run: "r",
21836 node: "implement",
21837 prompts: &prompts,
21838 cache,
21839 round: None,
21840 carry_seats: false,
21841 pauses: None,
21842 };
21843 wave(jobs, sem, &ctx, st, 0).await
21844 }
21845
21846 #[tokio::test]
21847 async fn verification_cache_wait_preserves_the_command_budget() {
21848 crate::run::pin_test_home();
21849 let tmp = tempfile::tempdir().unwrap();
21850 let cache = tmp.path().join("target");
21851 let guard = hold(&cache);
21852 let release = tokio::spawn(async move {
21853 tokio::time::sleep(Duration::from_millis(100)).await;
21854 guard.release();
21855 });
21856 let mut st = state(10);
21857 let budget = Duration::from_millis(50);
21858 let wait = repository_wait(&st, None);
21859 let (outcomes, _) = with_cache_lease(
21860 &mut st,
21861 Some(&cache),
21862 "gate",
21863 "gate",
21864 tmp.path(),
21865 "abc1234",
21866 budget,
21867 "test gate",
21868 &wait,
21869 |_, available| async move {
21870 assert_eq!(
21871 available, budget,
21872 "contention must not shorten verification"
21873 );
21874 (Vec::new(), false, Vec::new())
21875 },
21876 )
21877 .await
21878 .unwrap();
21879 release.await.unwrap();
21880 assert!(outcomes.is_empty(), "{outcomes:?}");
21881 assert!(
21882 st.events
21883 .iter()
21884 .any(|event| event.message.contains("waiting for"))
21885 );
21886 assert!(st.resource_wait.is_none());
21887 hold(&cache).release();
21889 }
21890
21891 #[tokio::test]
21892 async fn verification_cache_wait_expires_without_starting_commands() {
21893 crate::run::pin_test_home();
21894 let tmp = tempfile::tempdir().unwrap();
21895 let cache = tmp.path().join("target");
21896 let _held = hold(&cache);
21897 let mut st = state(0);
21898 let wait = repository_wait(&st, None);
21899 let (outcomes, _) = with_cache_lease(
21900 &mut st,
21901 Some(&cache),
21902 "gate",
21903 "gate",
21904 tmp.path(),
21905 "abc1234",
21906 Duration::from_secs(60),
21907 "test gate",
21908 &wait,
21909 |_, _| async { panic!("commands must not start without the cache lease") },
21910 )
21911 .await
21912 .unwrap();
21913 assert_eq!(outcomes.len(), 1);
21914 assert!(outcomes[0].resource_blocked);
21915 assert!(st.resource_wait.is_none());
21916 assert!(matches!(
21917 st.resource_block.unwrap().cause,
21918 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21919 ));
21920 }
21921
21922 #[tokio::test]
21923 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21924 crate::run::pin_test_home();
21925 let tmp = tempfile::tempdir().unwrap();
21926 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21927 let cache = tmp.path().join("cache");
21928 let mut state = state(21600);
21929 let wait = repository_wait(&state, None);
21930 let (outcomes, _) = with_cache_lease(
21931 &mut state,
21932 Some(&cache),
21933 "gate",
21934 "gate",
21935 tmp.path(),
21936 "head",
21937 Duration::from_secs(60),
21938 "gate",
21939 &wait,
21940 |_, _| async { panic!("freshness failure must not start verification") },
21941 )
21942 .await
21943 .unwrap();
21944 assert_eq!(outcomes.len(), 1);
21945 assert!(outcomes[0].resource_blocked);
21946 let block = state.resource_block.unwrap();
21947 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21948 assert!(!block.description().contains("timed out"));
21949 }
21950
21951 #[tokio::test]
21952 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21953 crate::run::pin_test_home();
21954 let tmp = tempfile::tempdir().unwrap();
21955 let cache = tmp.path().join("cache");
21956 let _held = hold(&cache);
21957 let mut state = state(21600);
21958 let pause = Pause::new();
21959 let interrupt = Pause::new();
21960 let cause = ParkCause::Operator;
21961 pause.park_for(cause.clone());
21962 let prompts = Prompts::default();
21963 let ctx = WaveCtx {
21964 run: "test",
21965 node: "fix",
21966 prompts: &prompts,
21967 cache: Some(&cache),
21968 round: None,
21969 carry_seats: false,
21970 pauses: Some((&pause, &interrupt)),
21971 };
21972 let result = wave(
21973 vec![job(
21974 sh("writer", "exit 99"),
21975 tmp.path(),
21976 Duration::from_secs(1),
21977 )],
21978 Arc::new(Semaphore::new(1)),
21979 &ctx,
21980 &mut state,
21981 0,
21982 )
21983 .await;
21984 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21985 assert_eq!(result[0].1.turns, 0);
21986 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21987 assert_eq!(
21988 state.resource_block.unwrap().cause,
21989 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21990 );
21991 assert!(state.active.is_empty());
21992 }
21993
21994 #[tokio::test]
21995 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21996 crate::run::pin_test_home();
21997 let tmp = tempfile::tempdir().unwrap();
21998 let cache = tmp.path().join("target-a");
21999 let guard = hold(&cache);
22000 tokio::spawn(async move {
22002 tokio::time::sleep(Duration::from_millis(1500)).await;
22003 guard.release();
22004 });
22005 let mut st = state(60);
22006 let results = run(
22007 vec![job(
22008 sh("alpha", "echo ok"),
22009 tmp.path(),
22010 Duration::from_secs(1),
22011 )],
22012 Arc::new(Semaphore::new(1)),
22013 Some(&cache),
22014 &mut st,
22015 )
22016 .await;
22017 assert!(
22018 matches!(results[0].2, AgentOutcome::Ok(_)),
22019 "the wait must not be charged to the agent"
22020 );
22021 assert!(st.resource_block.is_none());
22022 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
22023 assert!(st.resource_wait.is_none() && st.active.is_empty());
22024 }
22025
22026 #[tokio::test]
22027 #[cfg(unix)]
22028 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
22029 use std::os::fd::AsRawFd as _;
22030 crate::run::pin_test_home();
22031 let tmp = tempfile::tempdir().unwrap();
22032 crate::git::git(tmp.path(), &["init"]).await.unwrap();
22033 let common = tmp.path().join(".git");
22034 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
22035 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
22036 assert_eq!(
22037 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
22038 0
22039 );
22040 let mut st = state(1);
22041 st.config.coordination = crate::config::test_coordination();
22042 let mut reader = job(
22043 sh("reader", "echo must-not-run"),
22044 tmp.path(),
22045 Duration::from_secs(30),
22046 );
22047 reader.allow_write = false;
22048 let results = run(
22049 vec![reader.clone()],
22050 Arc::new(Semaphore::new(1)),
22051 None,
22052 &mut st,
22053 )
22054 .await;
22055 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
22056 assert_eq!(results[0].1.turns, 0);
22057 assert!(FailClass::of(&results[0].2).is_none());
22058 assert_eq!(
22059 st.resource_block.as_ref().unwrap().kind,
22060 "repository coordination"
22061 );
22062 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
22063 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22064 drop(external);
22065 let editor = crate::repo_lock::SeatLease::for_checkout(
22066 tmp.path(),
22067 true,
22068 crate::config::test_coordination().as_ref(),
22069 )
22070 .await
22071 .unwrap();
22072 let pause = Pause::new();
22073 let interrupt = Pause::new();
22074 let prompts = Prompts::default();
22075 let ctx = WaveCtx {
22076 run: "r",
22077 node: "review",
22078 prompts: &prompts,
22079 cache: None,
22080 round: None,
22081 carry_seats: false,
22082 pauses: Some((&pause, &interrupt)),
22083 };
22084 let (results, ()) = tokio::join!(
22085 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
22086 async {
22087 tokio::time::sleep(Duration::from_millis(50)).await;
22088 pause.park();
22089 }
22090 );
22091 assert!(
22092 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
22093 );
22094 assert_eq!(results[0].1.turns, 0);
22095 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
22096 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
22097 drop(editor);
22098 }
22099
22100 #[tokio::test]
22101 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
22102 crate::run::pin_test_home();
22103 let tmp = tempfile::tempdir().unwrap();
22104 let cache = tmp.path().join("target-b");
22105 let _held = hold(&cache);
22106 let mut st = state(1);
22107 let results = run(
22108 vec![job(
22109 sh("alpha", "echo ok"),
22110 tmp.path(),
22111 Duration::from_secs(1),
22112 )],
22113 Arc::new(Semaphore::new(1)),
22114 Some(&cache),
22115 &mut st,
22116 )
22117 .await;
22118 let (_, seat, out) = &results[0];
22119 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
22120 assert!(wave_blocked(&results));
22121 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
22122 assert_eq!(seat.turns, 0, "nothing was asked");
22123 assert!(
22124 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
22125 "no agent was spawned"
22126 );
22127 assert!(FailClass::of(out).is_none(), "not a failure class");
22128 let block = st.resource_block.as_ref().expect("recorded for the resume");
22129 assert_eq!(block.kind, "shared build cache");
22130 assert!(matches!(
22131 block.cause,
22132 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
22133 ));
22134 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
22135 assert!(st.active.is_empty());
22136 }
22137
22138 #[tokio::test]
22139 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
22140 crate::run::pin_test_home();
22141 let tmp = tempfile::tempdir().unwrap();
22142 let mut st = state(1);
22143 let results = run(
22144 vec![job(
22145 sh("alpha", "echo ok"),
22146 tmp.path(),
22147 Duration::from_secs(1),
22148 )],
22149 Arc::new(Semaphore::new(0)),
22150 None,
22151 &mut st,
22152 )
22153 .await;
22154 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
22155 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
22156 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
22157 }
22158
22159 #[tokio::test]
22160 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
22161 crate::run::pin_test_home();
22162 let tmp = tempfile::tempdir().unwrap();
22163 let mut st = state(60);
22164 let results = run(
22165 vec![job(
22166 sh("alpha", "sleep 30"),
22167 tmp.path(),
22168 Duration::from_secs(1),
22169 )],
22170 Arc::new(Semaphore::new(1)),
22171 None,
22172 &mut st,
22173 )
22174 .await;
22175 let out = &results[0].2;
22176 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
22177 let class = FailClass::of(out).expect("a failure");
22178 assert_eq!(class, FailClass::Timeout);
22179 assert!(should_hand_over(None, &class), "existing handover policy");
22180 assert!(st.resource_block.is_none());
22181 }
22182}