1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615impl Runner {
616 pub async fn start(
618 repo: &Path,
619 instruction: String,
620 config: Config,
621 origin: Origin,
622 ) -> Result<Self> {
623 Self::start_naming(repo, instruction, "", config, origin).await
624 }
625
626 pub async fn start_naming(
629 repo: &Path,
630 instruction: String,
631 _also_scan: &str,
632 config: Config,
633 origin: Origin,
634 ) -> Result<Self> {
635 let repo = git::toplevel(repo).await?;
636 let missing = agent::missing_programs(&config.agents);
637 if !missing.is_empty() {
638 bail!(
639 "these agent programs are not on PATH: {}. Fix the roster in \
640 magi.toml or install them.",
641 missing.join(", ")
642 );
643 }
644 let base_branch =
645 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646 .await?;
647 let fetched_base = resolve_base(
648 &repo,
649 &base_branch,
650 &config.merge.remote,
651 config.coordination.as_ref(),
652 )
653 .await?;
654 let base_commit = fetched_base.commit.clone();
655 let roles = config.resolve_roles()?;
656 let max_parallel = config.graph.max_parallel.max(1);
657 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658 state.fetched_base = Some(fetched_base);
662 state.origin = Some(origin);
663 state.reserve()?;
664 state.event("start", format!("run {} created", state.id));
665 state.save()?;
666 Ok(Self {
667 state,
668 roles,
669 sem: Arc::new(Semaphore::new(max_parallel)),
670 pause: Pause::new(),
671 interrupt: Pause::new(),
672 })
673 }
674
675 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689 Self::review_taking_over(repo, branch, config, None, origin).await
690 }
691
692 pub async fn review_taking_over(
699 repo: &Path,
700 branch: &str,
701 config: Config,
702 takeover: Option<crate::handover::Takeover>,
703 origin: Origin,
704 ) -> Result<Self> {
705 let repo = git::toplevel(repo).await?;
706 let missing = agent::missing_programs(&config.agents);
707 if !missing.is_empty() {
708 bail!(
709 "these agent programs are not on PATH: {}. Fix the roster in \
710 magi.toml or install them.",
711 missing.join(", ")
712 );
713 }
714 let base_branch =
715 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716 .await?;
717 if base_branch == branch {
718 bail!("`{branch}` is the base branch; there is nothing to review against");
719 }
720 let fetched_base = resolve_base(
721 &repo,
722 &base_branch,
723 &config.merge.remote,
724 config.coordination.as_ref(),
725 )
726 .await?;
727 let base_commit = fetched_base.commit.clone();
728
729 let roles = config.resolve_roles()?;
730 let max_parallel = config.graph.max_parallel.max(1);
731 let mut state = RunState::new(
732 repo.clone(),
733 base_branch,
734 base_commit.clone(),
735 String::new(),
736 config,
737 );
738 state.fetched_base = Some(fetched_base);
739 state.origin = Some(origin);
740 state.reserve()?;
741
742 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749 earlier: Vec::new(),
750 home: crate::run::home(),
751 choice: None,
752 task_instruction: None,
753 });
754 let wait = repository_wait(&state, None);
755 let coordination_config = state.config.coordination.clone();
756 let coordination =
757 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758 .await?;
759 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760 if let Some(released) = &released {
761 state.event(
762 "release",
763 format!(
764 "took `{branch}` over from run {}: its worktree was released: {}",
765 crate::run::short_of(&released.old_id),
766 released.audit
767 ),
768 );
769 }
770 state.foreign_checkout = crate::handover::preflight_review(
774 &repo,
775 branch,
776 &takeover.home,
777 &state.config.merge.remote,
778 )
779 .await?;
780 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
781 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
782 }
783 if let Some(choice) = takeover.choice.as_ref()
784 && let Err(e) =
785 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
786 .await
787 {
788 if let Some(released) = &released {
789 released.restore(&repo, branch).await;
790 }
791 return Err(e.context("applying the owner's answer about the diverged branch"));
792 }
793 for id in takeover.earlier.iter().rev() {
799 if let Ok(old) = RunState::load_under(id, &takeover.home)
800 && let Some(head) = old.pr_head.clone()
801 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
802 {
803 state.pr_head = Some(head);
804 break;
805 }
806 }
807 drop(coordination);
808 let opened = Self::open_review(
809 &repo,
810 branch,
811 state,
812 roles,
813 max_parallel,
814 base_commit,
815 &takeover,
816 )
817 .await;
818 if opened.is_err()
819 && let Some(released) = &released
820 {
821 let _coordination = crate::repo_lock::SeatLease::operation_with(
822 &repo,
823 coordination_config.as_ref(),
824 &wait,
825 )
826 .await?;
827 released.restore(&repo, branch).await;
828 }
829 opened
830 }
831
832 pub async fn recover_pr(
834 repo: &Path,
835 task: &str,
836 selection: crate::pr_recovery::Selection,
837 mut config: Config,
838 ) -> Result<Self> {
839 selection.configure(&mut config)?;
840 selection.check_task(task, &selection.instruction)?;
841 let base_branch =
842 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
843 .await?;
844 let fetched_base = resolve_base(
845 repo,
846 &base_branch,
847 &config.merge.remote,
848 config.coordination.as_ref(),
849 )
850 .await?;
851 let base_commit = fetched_base.commit.clone();
852 selection
853 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
854 .await?;
855 Box::pin(selection.verify_source(repo)).await?;
856 if let Some(proof) = &selection.source_proof {
857 anyhow::ensure!(
858 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
859 "source rebase base is outside the fetched landing base"
860 );
861 }
862 let roles = config.resolve_roles()?;
863 let max_parallel = config.graph.max_parallel.max(1);
864 let mut state = RunState::new(
865 repo.to_path_buf(),
866 base_branch,
867 base_commit.clone(),
868 selection.instruction.clone(),
869 config,
870 );
871 state.fetched_base = Some(fetched_base);
872 state.origin = Some(crate::run::Origin::queue(task));
873 state.pr_head = Some(selection.branch.clone());
874 state.pr_recovery = Some(selection.clone());
875 state.reserve()?;
876 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
877 state.discard_reserved();
879 return Err(e);
880 }
881 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
882 }
883
884 async fn open_selected(
885 repo: &Path,
886 state: RunState,
887 roles: ResolvedRoles,
888 max_parallel: usize,
889 base_commit: String,
890 selection: crate::pr_recovery::Selection,
891 ) -> Result<Self> {
892 let mut state = state;
893 let branch = state.branch_for('A');
894 let wait = repository_wait(&state, None);
895 let prepared = async {
896 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
897 git::git(repo, &["branch", &branch, selection.source()]).await?;
898 drop(coordination);
899 anyhow::Ok(())
900 }
901 .await;
902 if let Err(e) = prepared {
903 state.discard_reserved();
906 return Err(e);
907 }
908 state.event(
909 "recovery",
910 format!(
911 "selected implementation {} for {} (public lease {})",
912 selection.source(),
913 selection.url,
914 selection.head
915 ),
916 );
917 let takeover = crate::handover::Takeover {
918 earlier: Vec::new(),
919 home: crate::run::home(),
920 choice: None,
921 task_instruction: Some(selection.instruction.clone()),
922 };
923 if let Err(e) = state.save() {
924 state.discard_reserved();
925 return Err(e);
926 }
927 let id = state.id.clone();
928 let opened = Self::open_review(
929 repo,
930 &branch,
931 state,
932 roles,
933 max_parallel,
934 base_commit,
935 &takeover,
936 )
937 .await;
938 match opened {
939 Ok(runner) => Ok(runner),
940 Err(e) => {
941 if let Ok(mut partial) = RunState::load(&id) {
942 partial.status = RunStatus::Blocked;
943 partial.driver_exited = true;
944 partial.event(
945 "recovery",
946 format!("startup failed; retained source and candidate {branch}: {e:#}"),
947 );
948 partial.save()?;
949 }
950 Err(crate::pr_recovery::StartupFailed {
951 run: id,
952 cause: format!("{e:#}"),
953 }
954 .into())
955 }
956 }
957 }
958
959 async fn inherit(
964 repo: &Path,
965 branch: &str,
966 base_commit: &str,
967 takeover: &crate::handover::Takeover,
968 ) -> Option<crate::recovery::RecoveryRecord> {
969 use crate::recovery::{Relation, assess};
970 let newest = takeover.earlier.last()?;
971 let mut chosen = None;
972 for id in takeover.earlier.iter().rev() {
973 if let Ok(old) = RunState::load_under(id, &takeover.home)
974 && !old.reviews.is_empty()
975 && (old.winner().is_some_and(|w| w.branch == branch)
976 || old.pr_head.as_deref() == Some(branch))
977 {
978 chosen = Some(old);
979 break;
980 }
981 }
982 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
983 .await
984 .unwrap_or_default();
985 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
986 Some(last) if last.head == tip => Relation::Same,
987 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
988 _ => Relation::Diverged,
989 };
990 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
991 Some(assess(from, chosen.as_ref(), base_commit, relation))
992 }
993
994 async fn open_review(
997 repo: &Path,
998 branch: &str,
999 mut state: RunState,
1000 roles: ResolvedRoles,
1001 max_parallel: usize,
1002 base_commit: String,
1003 takeover: &crate::handover::Takeover,
1004 ) -> Result<Self> {
1005 let wait = repository_wait(&state, None);
1006 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1007 if state.foreign_checkout.is_some() {
1008 if state.pr_head.is_none() {
1010 state.pr_head = Some(branch.to_owned());
1011 }
1012 } else if state.pr_recovery.is_none() {
1013 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1014 }
1015 let start = review_base(
1018 repo,
1019 &state.config.merge.remote,
1020 &state.base_branch,
1021 &base_commit,
1022 branch,
1023 )
1024 .await;
1025 let log = git::log_oneline(repo, &start, branch)
1026 .await
1027 .unwrap_or_default();
1028 let log_text = if log.trim().is_empty() {
1029 "(no commit messages)"
1030 } else {
1031 log.trim()
1032 };
1033 let mut instruction = format!(
1034 "Review the work already on branch `{branch}`. There is no task \
1035 statement: what the change claims to do is whatever its commits \
1036 say.\n\n{log_text}"
1037 );
1038 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1042 if let Some(task) = takeover
1043 .task_instruction
1044 .as_deref()
1045 .filter(|t| !t.trim().is_empty())
1046 && rec.reviewed_head.is_some()
1047 {
1048 instruction = format!(
1049 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1050 task.trim()
1051 );
1052 rec.criteria_inherited = true;
1053 }
1054 state.event("recovery", rec.summary());
1055 state.recovery = Some(rec);
1056 }
1057 if let Some(selected) = &state.pr_recovery {
1058 instruction = selected.instruction.clone();
1059 }
1060 state.task_instruction = takeover.task_instruction.clone();
1061 state.instruction = instruction;
1062 state.reviewed_commits = Some(
1063 git::subjects(repo, &start, branch)
1064 .await
1065 .unwrap_or_default(),
1066 );
1067
1068 for id in &takeover.earlier {
1071 if let Ok(old) = RunState::load_under(id, &takeover.home)
1072 && old.repo == state.repo
1073 && (old.pr_head.as_deref() == Some(branch)
1074 || old.winner().is_some_and(|w| w.branch == branch))
1075 {
1076 for hold in old.branch_owner_holds {
1077 if !state.branch_owner_holds.contains(&hold) {
1078 state.event(
1079 "approval-policy",
1080 format!(
1081 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1082 hold.control.key(),
1083 hold.file,
1084 hold.head
1085 ),
1086 );
1087 state.branch_owner_holds.push(hold);
1088 }
1089 }
1090 }
1091 }
1092 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1096 let tracking = format!("{}/{branch}", state.config.merge.remote);
1097 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1100 let mut head = checkout.synced_head.clone();
1101 if matches!(fetched, Ok(out) if out.ok()) {
1102 let remote = git::rev_parse(repo, &tracking).await?;
1103 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1104 head = remote;
1105 }
1106 }
1107 head
1108 } else {
1109 git::rev_parse(repo, branch).await?
1110 };
1111 state.observe_branch_owner_holds(&reviewed_head).await;
1112
1113 let worktree = state.worktree_root().join("under-review");
1116 if let Some(parent) = worktree.parent() {
1117 tokio::fs::create_dir_all(parent).await.ok();
1118 }
1119 let path = worktree.to_string_lossy().to_string();
1120 let review_branch = if state.foreign_checkout.is_some() {
1121 let shadow = format!("magi/{}/review", state.short());
1122 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1123 shadow
1124 } else {
1125 branch.to_owned()
1126 };
1127 git::git(repo, &["worktree", "add", &path, &review_branch])
1128 .await
1129 .with_context(|| {
1130 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1131 })?;
1132
1133 let setup = match ensure_setup_config(&mut state, repo).await {
1134 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1135 Err(e) => Err(e),
1136 };
1137 if let Err(e) = setup {
1138 git::worktree_remove_clean(repo, &worktree).await.ok();
1139 return Err(e);
1140 }
1141 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1142 .await
1143 .unwrap_or(0);
1144 if commits == 0 {
1145 git::worktree_remove_clean(repo, &worktree).await.ok();
1146 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1147 }
1148 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1149 .await
1150 .map(|f| f.len())
1151 .unwrap_or(0);
1152 if files == 0
1153 && let (Ok(head_tree), Ok(base_tree)) = (
1154 git::tree_of(&worktree, "HEAD").await,
1155 git::tree_of(&worktree, &base_commit).await,
1156 )
1157 && head_tree == base_tree
1158 {
1159 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1160 git::worktree_remove_clean(repo, &worktree).await.ok();
1161 bail!(
1162 "`{branch}` at {} has a tree identical to base {}; this usually means \
1163 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1164 against `{}/{branch}`) rather than an empty change",
1165 short(&head),
1166 short(&base_commit),
1167 state.config.merge.remote
1168 );
1169 }
1170 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1171 .await
1172 .unwrap_or_default();
1173
1174 state.candidates.push(Candidate {
1175 index: 0,
1176 label: 'A',
1177 agent: EXISTING_BRANCH.to_owned(),
1180 branch: review_branch,
1181 worktree,
1182 summary: String::new(),
1183 stat,
1184 files,
1185 commits,
1186 empty: false,
1187 failed: None,
1188 verified_noop: None,
1189 duration_ms: 0,
1190 folded: false,
1191 });
1192 state.tally = Some(Tally {
1193 first_choice: BTreeMap::from([('A', 0)]),
1194 borda: BTreeMap::new(),
1195 winner: 'A',
1196 rankings: 0,
1197 unanimous_initial: false,
1198 deliberated: false,
1199 changed_votes: 0,
1200 unanimous_final: false,
1201 tie_break: None,
1202 judges: 0,
1206 present: 0,
1207 quorum: 0,
1208 met_quorum: true,
1209 uncontested: Some("review-only run: nothing competed".to_owned()),
1210 });
1211 state.status = RunStatus::Reviewing;
1212 state.prep_complete = Some(true);
1213 state.event(
1214 "start",
1215 format!(
1216 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1217 state.id
1218 ),
1219 );
1220 state.save()?;
1221 Ok(Self {
1222 state,
1223 roles,
1224 sem: Arc::new(Semaphore::new(max_parallel)),
1225 pause: Pause::new(),
1226 interrupt: Pause::new(),
1227 })
1228 }
1229
1230 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1233 Self::authorize_base_recovery_request(home, id, task, false).await
1234 }
1235
1236 pub(crate) async fn authorize_base_recovery_from_resumption(
1238 home: &Path,
1239 id: &str,
1240 task: &str,
1241 ) -> Result<()> {
1242 Self::authorize_base_recovery_request(home, id, task, true).await
1243 }
1244
1245 async fn authorize_base_recovery_request(
1246 home: &Path,
1247 id: &str,
1248 task: &str,
1249 resumption: bool,
1250 ) -> Result<()> {
1251 let _owner = crate::pr_head::owner_lock(home, id)?;
1252 let mut state = RunState::load_under(id, home)?;
1253 crate::pr_head::ensure_owner(&state)?;
1254 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1255 anyhow::ensure!(
1256 queued.runs.last().is_some_and(|run| run == id)
1257 && queued.status != crate::queue::TaskStatus::Done
1258 && !queued.fresh_start
1259 && queued.review_branch.is_none(),
1260 "run task ownership changed"
1261 );
1262 anyhow::ensure!(
1263 state
1264 .origin
1265 .as_ref()
1266 .and_then(|o| o.task.as_deref())
1267 .is_none_or(|owner| owner == task),
1268 "run belongs to another task"
1269 );
1270 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1271 anyhow::ensure!(
1272 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1273 "run is live or its ownership is unknown"
1274 );
1275 let _execution = crate::driver::maintenance(home, id)?;
1278 anyhow::ensure!(
1279 state.status.resumable()
1280 && state.released_to.is_none()
1281 && state.retired_to.is_none()
1282 && state.pr_taken_over_by.is_none(),
1283 "run ownership is final"
1284 );
1285 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1286 anyhow::ensure!(
1287 !state.pr_publication_pending
1288 && !crate::handover::landing_checkpoint(&state)
1289 && state.merge_decision.is_none(),
1290 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1291 );
1292 let winner = state.winner().context("run has no retained winner")?;
1293 anyhow::ensure!(
1294 git::is_clean(&winner.worktree).await?,
1295 "retained worktree has uncommitted changes"
1296 );
1297 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1298 anyhow::ensure!(
1299 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1300 "retained worktree does not match the winner branch"
1301 );
1302 state.record_base_sync_stop(&head);
1303 state.save()?;
1304 if resumption {
1305 anyhow::ensure!(
1306 crate::daemon::compatible_run_instruction(&state, &queued),
1307 "resumption instruction does not match the retained run"
1308 );
1309 let approval = queued
1310 .resumption
1311 .as_ref()
1312 .context("no operator resumption")?;
1313 anyhow::ensure!(
1314 approval.instruction == queued.instruction,
1315 "resumption approval does not match the current task instruction"
1316 );
1317 state.authorize_base_sync_resumption(task, approval)?;
1318 } else {
1319 state.authorize_base_sync(task)?;
1320 }
1321 state.event(
1322 "base-recovery",
1323 "owner authorized one additional base rebase; fresh verification required",
1324 );
1325 state.save()?;
1326 Ok(())
1327 }
1328
1329 pub fn resume(id: &str) -> Result<Self> {
1331 let state = RunState::load(id)?;
1332 crate::pr_head::ensure_owner(&state)?;
1333 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1334 bail!(
1335 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1336 );
1337 }
1338 if let Some(to) = &state.released_to {
1339 bail!(
1340 "run {} cannot be resumed: its worktree was released to run {}",
1341 state.short(),
1342 crate::run::short_of(to)
1343 );
1344 }
1345 let roles = state.config.resolve_roles()?;
1346 if let Some(id) = &state.config.graph.selected_implementer
1347 && state.candidates.iter().any(|c| c.agent != *id)
1348 {
1349 bail!(
1350 "replacement implementer pin does not match the saved candidates; task must stay held"
1351 );
1352 }
1353 let max_parallel = state.config.graph.max_parallel.max(1);
1354 Ok(Self {
1355 state,
1356 roles,
1357 sem: Arc::new(Semaphore::new(max_parallel)),
1358 pause: Pause::new(),
1359 interrupt: Pause::new(),
1360 })
1361 }
1362
1363 pub async fn execute(&mut self) -> Result<()> {
1370 if let Some(selection) = &self.state.pr_recovery
1371 && (selection.source_proof.is_some() || selection.source_run.is_some())
1372 {
1373 Box::pin(selection.verify_source(&self.state.repo)).await?;
1374 }
1375 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1378 || self.state.land_approval.is_some()
1379 || self.state.merge_decision.is_some()
1380 || !self.state.merge_repairs.is_empty()
1381 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1382 {
1383 let claim = FixClaim::acquire(&self.state.dir())?;
1384 let disk = RunState::load(&self.state.id)?;
1385 let checkpoint = |s: &RunState| {
1386 (
1387 s.merge_repairs.len(),
1388 s.merge_repairs
1389 .last()
1390 .map(|r| (r.question.clone(), r.phase)),
1391 s.land_approval
1392 .as_ref()
1393 .map(|a| (a.question.clone(), a.head.clone())),
1394 )
1395 };
1396 if checkpoint(&disk) != checkpoint(&self.state) {
1397 bail!(
1398 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1399 );
1400 }
1401 Some(claim)
1402 } else {
1403 None
1404 };
1405 let mut execution = self.claim_driver().await?;
1406 let runner = &mut *execution.runner;
1407 let wait = runner.repository_wait();
1408 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1411 let result = match result {
1412 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1413 record_repository_block(&mut runner.state, &e);
1414 runner.park_here()?;
1415 Ok(())
1416 }
1417 other => other,
1418 };
1419 execution.driver.finish(&mut runner.state).await?;
1420 let ended = if result.is_err() {
1421 Some({
1422 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1423 if let Err(error) = &result {
1424 notice.message = format!(
1425 "Run {} stopped with an error: {}",
1426 runner.state.id,
1427 error.to_string().lines().next().unwrap_or("unknown error")
1428 );
1429 }
1430 notice
1431 })
1432 } else {
1433 crate::notices::run_ended(&runner.state)
1434 };
1435 if let Some(notice) = ended {
1436 crate::notices::raise(notice);
1437 }
1438 result
1439 }
1440
1441 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1442 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1443 }
1444
1445 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1446 let wait = self.repository_wait();
1447 coordinate_operation(&mut self.state, cwd, &wait).await
1448 }
1449
1450 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1452 let owner = match crate::pr_head::owner_lock_wait(
1453 &crate::run::home(),
1454 &self.state.id,
1455 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1456 )
1457 .await
1458 {
1459 Ok(owner) => owner,
1460 Err(error) => {
1461 let detail = format!("worktree maintenance or ownership update: {error:#}");
1462 self.state.resource_block = Some(crate::run::ResourceBlock {
1466 cause: crate::run::ResourceBlockCause::Other,
1467 node: "start".into(),
1468 kind: "worktree maintenance".into(),
1469 detail: detail.clone(),
1470 at: Timestamp::now(),
1471 });
1472 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1473 }
1474 };
1475 let disk = RunState::load(&self.state.id)?;
1476 crate::pr_head::ensure_owner(&disk)?;
1477 let claimed =
1478 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1479 anyhow::ensure!(
1484 disk.liveness(claimed) != crate::run::Liveness::Live,
1485 "run already has a live driver; wait for its execution to end"
1486 );
1487 if let Some(to) = &disk.released_to {
1488 bail!(
1489 "run {} cannot continue: its worktree was released to run {}",
1490 self.state.short(),
1491 crate::run::short_of(to)
1492 );
1493 }
1494 anyhow::ensure!(
1495 serde_json::to_value(&disk.base_sync_recovery)?
1496 == serde_json::to_value(&self.state.base_sync_recovery)?,
1497 "base recovery changed since this run was loaded; reload the saved run"
1498 );
1499 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1500 let mut execution = DriverExecution {
1501 runner: self,
1502 driver,
1503 };
1504 let _owner = owner;
1507 let state = &mut execution.runner.state;
1508 state.parked = false;
1513 state.resource_wait = None;
1514 state.resource_block = None;
1515 state.clear_active();
1522 let pid = std::process::id();
1537 state.driver_pid = Some(pid);
1538 state.driver_started_at = crate::proc::process_started_at(pid);
1539 state.driver_exited = false;
1540 state.driver_claimed = Some(true);
1541 state.begin_pass();
1542 execution.driver.bind(state);
1543 state.save()?;
1544 execution.driver.record(state)?;
1545 Ok(execution)
1546 }
1547
1548 async fn execute_graph(&mut self) -> Result<()> {
1549 self.state.configure_recovery_resume()?;
1550 self.persist_candidates().await?;
1553 if self.state.base_sync_blocked() {
1556 self.sync_to_base().await?;
1557 if !self.state.base_sync_blocked()
1558 && self.state.status != RunStatus::AlreadyInBase
1559 && !self.resource_halt()?
1560 {
1561 self.finish_after_tally().await?;
1562 }
1563 return Ok(());
1564 }
1565 if self
1566 .state
1567 .merge_decision
1568 .as_ref()
1569 .is_some_and(|decision| !decision.closed)
1570 && self.state.status != RunStatus::Landing
1571 {
1572 if crate::merge_decision::resumable_hold(&self.state) {
1573 self.state.merge = Some(MergeOutcome {
1574 mode: MergeMode::Pr,
1575 ok: false,
1576 detail: self
1577 .state
1578 .merge_decision
1579 .as_ref()
1580 .unwrap()
1581 .target
1582 .pr
1583 .clone(),
1584 empty: false,
1585 });
1586 self.run_land().await?;
1587 self.settle_questions();
1588 return Ok(());
1589 }
1590 bail!(
1591 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1592 );
1593 }
1594 if self.state.config.graph.selected_implementer.is_some()
1595 && self.state.status == RunStatus::Blocked
1596 && self.state.tally.is_none()
1597 && self.state.candidate_preservation.is_empty()
1598 && self.state.interrupted_implementations.is_empty()
1599 {
1600 bail!(
1601 "selected implementation is held; explicitly release the owning task to retry it"
1602 );
1603 }
1604
1605 if self.state.merge_repairs.last().is_some_and(|r| {
1606 matches!(
1607 r.phase,
1608 crate::run::MergeRepairPhase::Requested
1609 | crate::run::MergeRepairPhase::Fixing
1610 | crate::run::MergeRepairPhase::Reviewing
1611 )
1612 }) {
1613 if self.repair_merge_review().await? {
1614 self.run_land().await?;
1615 }
1616 self.settle_questions();
1617 return Ok(());
1618 }
1619 if self.state.pr_publication_pending {
1622 self.merge().await?;
1623 if self.resource_halt()? {
1624 return Ok(());
1625 }
1626 self.state.save()?;
1627 self.settle_questions();
1628 return Ok(());
1629 }
1630 if self.state.status == RunStatus::Stalled {
1643 if self.recover_stall().await? {
1644 self.finish_after_tally().await?;
1645 } else {
1646 self.state.save()?;
1648 }
1649 return Ok(());
1650 }
1651 if self.state.status == RunStatus::Landing
1661 || crate::handover::landing_checkpoint(&self.state)
1662 {
1663 self.state.land_sync_pending = true;
1664 self.state.save()?;
1665 self.run_land().await?;
1666 self.settle_questions();
1671 return Ok(());
1672 }
1673 if self.state.github_text.is_some()
1678 && self.state.merge.is_none()
1679 && self.state.status == RunStatus::Gating
1680 {
1681 self.merge().await?;
1682 if self.resource_halt()? {
1683 return Ok(());
1684 }
1685 self.state.save()?;
1686 return Ok(());
1687 }
1688 if let Err(e) = self.prep().await {
1689 if e.is::<crate::repo_lock::ResourceBlocked>() {
1690 return Err(e);
1691 }
1692 self.state.status = RunStatus::Blocked;
1693 self.state
1694 .prep_failure
1695 .get_or_insert_with(|| format!("{e:#}"));
1696 self.state
1697 .event("prep", format!("preparation stopped: {e:#}"));
1698 self.state.save()?;
1699 return Err(e);
1700 }
1701 if self.park_here()? {
1702 return Ok(());
1703 }
1704 self.advise().await?;
1705 if self.park_here()? {
1706 return Ok(());
1707 }
1708 self.implement().await?;
1709 if self.park_here()? {
1710 return Ok(());
1711 }
1712 if self.state.status == RunStatus::VerifiedNoop {
1716 return Ok(());
1717 }
1718 self.judge().await?;
1719 if self.park_here()? {
1720 return Ok(());
1721 }
1722 self.deliberate().await?;
1723 if self.park_here()? {
1724 return Ok(());
1725 }
1726 self.vote().await?;
1727 if self.park_here()? {
1728 return Ok(());
1729 }
1730 self.tally()?;
1731 if self.state.status == RunStatus::Stalled {
1736 self.state.save()?;
1740 return Ok(());
1741 }
1742 self.finish_after_tally().await?;
1743 Ok(())
1744 }
1745
1746 fn stop_for_resource(&mut self) -> Result<()> {
1751 if self.pause.parked() || self.interrupt.parked() {
1754 self.park_here()?;
1755 return Ok(());
1756 }
1757 if self.state.resource_block.is_some() {
1758 self.state.status = RunStatus::Blocked;
1759 }
1760 self.state.save()
1761 }
1762
1763 fn resource_halt(&mut self) -> Result<bool> {
1765 if self.state.resource_block.is_none() {
1766 return Ok(false);
1767 }
1768 self.stop_for_resource()?;
1769 Ok(true)
1770 }
1771
1772 fn park_here(&mut self) -> Result<bool> {
1779 let parking = self.pause.parked() || self.interrupt.parked();
1780 if !parking && self.state.resource_block.is_some() {
1781 self.stop_for_resource()?;
1782 return Ok(true);
1783 }
1784 if !self.pause.parked() && !self.interrupt.parked() {
1789 return Ok(false);
1790 }
1791 if self.state.parked {
1792 return Ok(true);
1793 }
1794 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1795 if let Some(block) = &mut self.state.resource_block
1796 && matches!(
1797 block.cause,
1798 crate::run::ResourceBlockCause::Cancelled { .. }
1799 )
1800 {
1801 block.cause = crate::run::ResourceBlockCause::Cancelled {
1802 park: cause.clone(),
1803 };
1804 }
1805 let why = match cause.as_ref().map(ParkCause::label) {
1806 Some(reason) => format!(
1807 "parked after `{}` ({reason}) — resume to carry on from here",
1808 self.state.status.as_str()
1809 ),
1810 None => format!(
1811 "parked after `{}` — resume to carry on from here",
1812 self.state.status.as_str()
1813 ),
1814 };
1815 self.state.event("park", why);
1816 self.state
1817 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1818 self.state.parked = true;
1819 self.state.save()?;
1820 Ok(true)
1821 }
1822
1823 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1825 if self.state.prep_failure.is_none() {
1826 return Ok(());
1827 }
1828 let failed_at = self
1829 .state
1830 .events
1831 .iter()
1832 .rev()
1833 .find(|event| event.node == "prep")
1834 .map_or(self.state.updated_at, |event| event.at);
1835 anyhow::ensure!(
1836 approval_at > failed_at,
1837 "preparation retry requires a new explicit release after the failure"
1838 );
1839 self.state.prep_failure = None;
1840 self.state.prep_complete = Some(false);
1841 self.state
1842 .event("prep-retry", "operator approved retry of preparation");
1843 Ok(())
1844 }
1845
1846 pub fn on_pause(&mut self, pause: Pause) {
1848 self.pause = pause;
1849 }
1850
1851 pub fn watch_interrupt(&mut self, pause: Pause) {
1857 self.interrupt = pause;
1858 }
1859
1860 fn settle_questions(&mut self) {
1880 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1881 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1882 }
1883 }
1884
1885 async fn finish_after_tally(&mut self) -> Result<()> {
1888 self.graft().await?;
1891 if self.resource_halt()? {
1892 return Ok(());
1893 }
1894 self.fold_losers().await?;
1895 self.sync_to_base().await?;
1900 if self.hold_base_sync().await? || self.resource_halt()? {
1901 return Ok(());
1902 }
1903 if self.state.status == RunStatus::AlreadyInBase {
1904 return Ok(());
1905 }
1906 self.review_loop().await?;
1907 if self.resource_halt()? {
1908 return Ok(());
1909 }
1910 self.sync_to_base().await?;
1911 if self.hold_base_sync().await? || self.resource_halt()? {
1912 return Ok(());
1913 }
1914 if self.state.status == RunStatus::AlreadyInBase {
1915 return Ok(());
1916 }
1917 self.gate().await?;
1918 self.merge().await?;
1919 if self.resource_halt()? {
1920 return Ok(());
1921 }
1922 self.state.save()?;
1923 Ok(())
1924 }
1925
1926 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1933 let result = match ensure_setup_config(&mut self.state, repo).await {
1934 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1935 Err(e) => Err(e),
1936 };
1937 if let Err(e) = result {
1938 git::worktree_remove_clean(repo, wt).await.ok();
1939 if in_prep {
1940 self.rollback_prep(repo).await;
1941 }
1942 self.state.status = RunStatus::Blocked;
1943 self.state.event(
1944 "setup",
1945 format!("worktree setup failed in {}: {e:#}", wt.display()),
1946 );
1947 self.state.save()?;
1948 return Err(e);
1949 }
1950 Ok(())
1951 }
1952
1953 async fn rollback_prep(&mut self, repo: &Path) {
1955 let root = self.state.worktree_root();
1956 for c in std::mem::take(&mut self.state.candidates) {
1957 if matches!(
1958 git::worktree_remove_clean(repo, &c.worktree).await,
1959 Ok(true)
1960 ) {
1961 git::branch_delete(repo, &c.branch).await.ok();
1962 } else {
1963 self.state.candidates.push(c);
1964 }
1965 }
1966 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1967 for j in 1..=seats {
1968 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1969 .await
1970 .ok();
1971 }
1972 for k in 1..=self.state.config.graph.advisors {
1973 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1974 .await
1975 .ok();
1976 }
1977 }
1978
1979 async fn prep(&mut self) -> Result<()> {
1980 if let Some(reason) = &self.state.prep_failure {
1981 bail!(
1982 "candidate preparation previously failed: {reason}; correct the source before retrying"
1983 );
1984 }
1985 if self
1986 .state
1987 .prep_complete
1988 .unwrap_or(!self.state.candidates.is_empty())
1989 {
1990 return Ok(());
1991 }
1992 let repo = self.state.repo.clone();
1993 let _coordination = self.coordinate(&repo).await?;
1996 let base = self.state.base_commit.clone();
1997 let plan = self.state.sources.plan(&repo, &base).await?;
1998 self.rollback_prep(&repo).await;
1999 if !self.state.candidates.is_empty() {
2000 bail!(
2001 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2002 );
2003 }
2004 self.state.prep_complete = Some(false);
2005 self.state.status = RunStatus::Prep;
2006 self.state.save()?;
2007 let repo = self.state.repo.clone();
2008 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2009 let root = self.state.worktree_root();
2010 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2011
2012 let hooks_dir = self.state.dir().join("hooks");
2015 if self.state.config.blind.commit_msg_hook {
2016 std::fs::create_dir_all(&hooks_dir)
2017 .with_context(|| format!("create {}", hooks_dir.display()))?;
2018 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2019 let path = hooks_dir.join("commit-msg");
2020 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2021 make_executable(&path)?;
2022 git::acquire_worktree_config(&repo).await?;
2030 self.state.enabled_worktree_config = true;
2031 }
2032
2033 ensure_setup_config(&mut self.state, &repo).await?;
2036
2037 for (index, (spec, label)) in self
2038 .roles
2039 .implementers
2040 .clone()
2041 .into_iter()
2042 .zip(labels)
2043 .enumerate()
2044 {
2045 let branch = self.state.branch_for(label);
2046 let worktree = root.join(format!("cand-{label}"));
2047 self.state.event(
2051 "prep-intent",
2052 serde_json::json!({
2053 "branch": branch,
2054 "worktree": worktree,
2055 "start": start,
2056 "picks": plan.picks,
2057 })
2058 .to_string(),
2059 );
2060 self.state.save()?;
2061 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2062 if self.state.config.blind.commit_msg_hook {
2063 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2064 }
2065 git::local_exclude(&worktree, "/.magi/").await?;
2066 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2067 git::worktree_remove_clean(&repo, &worktree).await.ok();
2070 git::branch_delete(&repo, &branch).await.ok();
2071 self.rollback_prep(&repo).await;
2072 self.state.status = RunStatus::Blocked;
2073 self.state
2074 .event("prep", format!("worktree setup failed: {e:#}"));
2075 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2076 self.state.save()?;
2077 return Err(e);
2078 }
2079 self.state.candidates.push(Candidate {
2080 index,
2081 label,
2082 agent: spec.id.clone(),
2083 branch,
2084 worktree,
2085 summary: String::new(),
2086 stat: String::new(),
2087 files: 0,
2088 commits: 0,
2089 empty: false,
2090 failed: None,
2091 verified_noop: None,
2092 duration_ms: 0,
2093 folded: false,
2094 });
2095 self.state.save()?;
2096 for pick in &plan.picks {
2097 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2098 {
2099 self.state.status = RunStatus::Blocked;
2100 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2101 self.state
2102 .event("prep", format!("cannot apply referenced commit: {e}"));
2103 self.state.save()?;
2104 return Err(e);
2105 }
2106 }
2107 }
2108
2109 for j in 1..=self.roles.judges.len() {
2110 let wt = root.join(format!("judge-{j}"));
2111 if !wt.exists() {
2112 git::worktree_add_detached(&repo, &wt, &base).await?;
2113 self.setup_seat_worktree(&repo, &wt, true).await?;
2114 }
2115 }
2116
2117 if self.state.config.graph.advise {
2125 for k in 1..=self.state.config.graph.advisors {
2126 let wt = root.join(format!("advisor-{k}"));
2127 if !wt.exists() {
2128 git::worktree_add_detached(&repo, &wt, &base).await?;
2129 self.setup_seat_worktree(&repo, &wt, true).await?;
2130 }
2131 }
2132 }
2133
2134 let authors: Vec<&str> = self
2139 .roles
2140 .implementers
2141 .iter()
2142 .map(|a| a.id.as_str())
2143 .collect();
2144 let overlap: Vec<String> = self
2145 .roles
2146 .judges
2147 .iter()
2148 .enumerate()
2149 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2150 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2151 .collect();
2152 if !overlap.is_empty() {
2153 let note = format!(
2154 "{} also authored a candidate; blind, but the panel is less \
2155 independent than {} distinct agents would be",
2156 overlap.join(", "),
2157 self.roles.judges.len()
2158 );
2159 self.state.event("prep", note);
2160 }
2161
2162 self.state.event(
2163 "prep",
2164 format!(
2165 "{} candidates, {} judges, base {} ({})",
2166 self.state.candidates.len(),
2167 self.roles.judges.len(),
2168 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2169 self.state.base_branch
2170 ),
2171 );
2172 self.state.prep_complete = Some(true);
2173 self.state.status = RunStatus::Implementing;
2174 self.state.save()?;
2175 Ok(())
2176 }
2177
2178 async fn advise(&mut self) -> Result<()> {
2211 let implement_untouched = self
2212 .state
2213 .candidates
2214 .iter()
2215 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2216 if !self.state.config.graph.advise || self.state.advise_attempted {
2217 return Ok(());
2218 }
2219 if !implement_untouched {
2220 self.state.event(
2221 "advise",
2222 "skipping the design-deliberation stage: at least one \
2223 candidate already shows implementation progress, so this \
2224 run is past the point the stage exists to run before"
2225 .to_owned(),
2226 );
2227 self.state.advise_attempted = true;
2228 self.state.save()?;
2229 return Ok(());
2230 }
2231 let run_id = self.state.id.clone();
2232 let prompts = self.state.config.prompts.clone();
2233 let instruction = self.state.instruction.clone();
2234 let language = self.state.config.graph.language.clone();
2235 let root = self.state.worktree_root();
2236 let n = self.state.config.graph.advisors;
2237 let where_recorded = self.state.dir().join("run.json");
2238
2239 let seats = match self.state.config.advisors() {
2240 Ok(seats) if !seats.is_empty() => seats,
2241 Ok(_) => {
2242 self.state.event(
2243 "advise",
2244 format!(
2245 "[graph] advisors is 0; skipping the design-deliberation \
2246 stage and continuing without a synthesis brief (see {})",
2247 where_recorded.display()
2248 ),
2249 );
2250 self.state.advise_attempted = true;
2251 self.state.save()?;
2252 return Ok(());
2253 }
2254 Err(e) => {
2255 self.state.event(
2256 "advise",
2257 format!(
2258 "could not resolve advisor seats ({e:#}); continuing \
2259 without a design-deliberation brief (see {})",
2260 where_recorded.display()
2261 ),
2262 );
2263 self.state.advise_attempted = true;
2264 self.state.save()?;
2265 return Ok(());
2266 }
2267 };
2268
2269 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2270 let artifacts = agent::artifacts_dir(&self.state.dir());
2271 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2272
2273 let mut jobs = Vec::new();
2274 for (i, spec) in seats.iter().cloned().enumerate() {
2275 let seat_key = format!("advisor-{}", i + 1);
2276 let seat = self.seat(&seat_key, &spec.id);
2277 jobs.push(SeatJob {
2278 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2279 spec,
2280 seat,
2281 cwd: worktrees[i % worktrees.len()].clone(),
2282 timeout,
2283 allow_write: false,
2284 sessions: false,
2285 artifacts: artifacts.clone(),
2286 stem: seat_key,
2287 handover: None,
2288 });
2289 }
2290
2291 self.state.event(
2292 "advise",
2293 format!(
2294 "{} advisor seat(s) sketching a design in parallel",
2295 jobs.len()
2296 ),
2297 );
2298 let mut quota_losses = Vec::new();
2299 let cache = self.state.config.cache_dir();
2300 let ctx = WaveCtx {
2301 carry_seats: false,
2302 pauses: Some((&self.pause, &self.interrupt)),
2303 run: &run_id,
2304 node: "advise",
2305 prompts: &prompts,
2306 cache: cache.as_deref(),
2307 round: None,
2308 };
2309 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2310 let results = ask_json_wave::<Proposal>(
2311 jobs,
2312 Arc::clone(&self.sem),
2313 self.state.config.graph.retries,
2314 &advisor_roster,
2315 &ctx,
2316 &mut quota_losses,
2317 &mut self.state,
2318 &|p: &Proposal| p.validate(),
2319 )
2320 .await;
2321 if self.state.resource_block.is_some() {
2322 return self.stop_for_resource();
2323 }
2324 self.state.quota.extend(quota_losses);
2325
2326 let mut records = Vec::with_capacity(results.len());
2327 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2328 let agent_id = seat.agent.clone();
2329 self.state.seats.insert(seat.key.clone(), seat);
2330 match res {
2331 Ok((proposal, out)) => {
2332 self.state
2333 .event("advise", format!("advisor-{} proposed a design", i + 1));
2334 records.push(advise::AdvisorRecord::proposed(
2335 i + 1,
2336 agent_id,
2337 proposal,
2338 out.duration_ms,
2339 ));
2340 }
2341 Err(e) => {
2342 self.state.event(
2343 "advise",
2344 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2345 );
2346 records.push(advise::AdvisorRecord::failed(
2347 i + 1,
2348 agent_id,
2349 e.to_string(),
2350 ));
2351 }
2352 }
2353 }
2354
2355 let mut advice = advise::Advice {
2356 records,
2357 synthesis: None,
2358 };
2359 if advice.proposals().is_empty() {
2360 self.state.event(
2361 "advise",
2362 "no advisor produced a usable proposal; continuing without a \
2363 synthesis brief"
2364 .to_owned(),
2365 );
2366 } else {
2367 match self
2368 .synthesize_brief(
2369 &advice,
2370 &instruction,
2371 &language,
2372 &worktrees[0],
2373 &artifacts,
2374 &run_id,
2375 &prompts,
2376 cache.as_deref(),
2377 )
2378 .await
2379 {
2380 Ok(Some(text)) => {
2381 self.state.event(
2382 "advise",
2383 "synthesized a design brief for the implementer".to_owned(),
2384 );
2385 advice.synthesis = Some(text);
2386 }
2387 Ok(None) => {
2388 self.state.event(
2389 "advise",
2390 "the synthesis seat produced nothing usable; continuing \
2391 without a design brief"
2392 .to_owned(),
2393 );
2394 }
2395 Err(e) => {
2396 if e.is::<crate::repo_lock::ResourceBlocked>() {
2397 return Err(e);
2398 }
2399 self.state.event(
2400 "advise",
2401 format!("could not synthesize a design brief: {e:#}"),
2402 );
2403 }
2404 }
2405 }
2406 advise::apply_reflection(&mut advice);
2407
2408 self.state.advice = Some(advice);
2409 self.state.advise_attempted = true;
2410 self.state.save()?;
2411 Ok(())
2412 }
2413
2414 #[allow(clippy::too_many_arguments)]
2426 async fn synthesize_brief(
2427 &mut self,
2428 advice: &advise::Advice,
2429 instruction: &str,
2430 language: &str,
2431 cwd: &Path,
2432 artifacts: &Path,
2433 run_id: &str,
2434 prompts: &Prompts,
2435 cache: Option<&Path>,
2436 ) -> Result<Option<String>> {
2437 let chain = agent::pick_chain(
2438 &self.state.config.agents,
2439 self.state.config.roles.synthesizer.as_ref(),
2440 &agent::installed,
2441 "synthesizer",
2442 )?;
2443 let proposals = advice.proposals();
2444 let mut prompt = prompt::with_overlay(
2445 prompt::synthesize_brief(instruction, &proposals, language),
2446 prompts.overlay("advise"),
2447 );
2448 if cache.is_some() {
2449 prompt.push('\n');
2454 prompt.push_str(&prompt::build_cache_note("advise", false));
2455 }
2456 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2457 let mut last = None;
2462 for (n, spec) in chain.iter().enumerate() {
2463 if n > 0 {
2464 self.state
2465 .event("advise", format!("synthesis falling back to {}", spec.id));
2466 }
2467 let mut seat = self.seat("advise-synthesis", &spec.id);
2468 let wait = self.repository_wait();
2469 let outcome = wait
2470 .scope(agent::invoke(
2471 spec,
2472 &mut seat,
2473 &Invocation {
2474 coordination: self.state.config.coordination.as_ref(),
2475 fetched_base: None,
2476 cwd,
2477 prompt: &prompt,
2478 timeout,
2479 allow_write: false,
2480 sessions: false,
2481 artifacts,
2482 stem: &if n == 0 {
2483 "advise-synthesis".to_owned()
2484 } else {
2485 format!("advise-synthesis-{}", spec.id)
2486 },
2487 run: run_id,
2488 node: "advise",
2489 cache_dir: None,
2490 attachments: &[],
2491 writable: &[],
2492 },
2493 ))
2494 .await;
2495 if outcome.is_ok() {
2496 self.state.seats.insert(seat.key.clone(), seat);
2497 }
2498 let advance = agent::chain_advances(&outcome);
2499 last = Some(outcome);
2500 if !advance {
2501 break;
2502 }
2503 }
2504 let out = last.expect("a chain holds at least one agent")?;
2507 if !out.usable() {
2508 return Ok(None);
2509 }
2510 let text =
2511 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2512 Ok((!text.trim().is_empty()).then_some(text))
2513 }
2514
2515 async fn implement(&mut self) -> Result<()> {
2518 self.persist_candidates().await?;
2519 let run_id = self.state.id.clone();
2524 let prompts = self.state.config.prompts.clone();
2525 let todo: Vec<usize> = self
2526 .state
2527 .candidates
2528 .iter()
2529 .enumerate()
2530 .filter(|(_, c)| {
2531 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2532 && c.failed.is_none()
2533 && !c.empty
2534 })
2535 .map(|(i, _)| i)
2536 .collect();
2537 if todo.is_empty() {
2538 return self.after_implement();
2539 }
2540 self.state.status = RunStatus::Implementing;
2541
2542 let language = self.state.config.graph.language.clone();
2543 let github_language = self.state.config.graph.github_language.clone();
2544 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2545 let sessions = self.state.config.graph.sessions;
2546 let artifacts = agent::artifacts_dir(&self.state.dir());
2547 let brief = self
2551 .state
2552 .advice
2553 .as_ref()
2554 .and_then(|a| a.synthesis.as_deref())
2555 .map(str::to_owned);
2556 let attachments = self.state.attachments.clone();
2557
2558 let mut jobs = Vec::new();
2559 for &i in &todo {
2560 let (index, label, worktree) = {
2561 let c = &self.state.candidates[i];
2562 (c.index, c.label, c.worktree.clone())
2563 };
2564 let spec = self.roles.implementers[index].clone();
2565 let seat_key = format!("impl-{label}");
2566 let seat = self.seat(&seat_key, &spec.id);
2567 let instruction = seeded_instruction(&self.state);
2568 jobs.push(SeatJob {
2569 spec,
2570 seat,
2571 prompt: prompt::implement_in(
2572 &instruction,
2573 &worktree.to_string_lossy(),
2574 &language,
2575 &github_language,
2576 brief.as_deref(),
2577 &attachments,
2578 ),
2579 cwd: worktree,
2580 timeout,
2581 allow_write: true,
2582 sessions,
2583 artifacts: artifacts.clone(),
2584 stem: format!("impl-{label}"),
2585 handover: None,
2586 });
2587 }
2588
2589 self.state.event(
2590 "implement",
2591 format!("{} candidates in parallel", jobs.len()),
2592 );
2593 let mut sent = jobs.clone();
2599 let cache = self.state.config.cache_dir();
2600 let ctx = WaveCtx {
2601 carry_seats: false,
2602 pauses: Some((&self.pause, &self.interrupt)),
2603 run: &run_id,
2604 node: "implement",
2605 prompts: &prompts,
2606 cache: cache.as_deref(),
2607 round: None,
2608 };
2609 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2610 if !wave_blocked(&results) {
2611 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2612 .await;
2613 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2614 .await?;
2615 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2616 .await;
2617 }
2618 if !results.is_empty()
2621 && results
2622 .iter()
2623 .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
2624 {
2625 let losses: Vec<_> = results
2626 .iter()
2627 .map(|(_, seat, out)| {
2628 let AgentOutcome::Quota(o) = out else {
2629 unreachable!()
2630 };
2631 QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
2632 })
2633 .collect();
2634 self.state.quota.extend(losses.clone());
2635 for (_, seat, _) in results {
2636 self.state.seats.insert(seat.key.clone(), seat);
2637 }
2638 for &i in &todo {
2639 let label = self.state.candidates[i].label;
2640 self.state.interrupted_implementations.insert(label);
2641 self.state
2642 .candidate_preservation
2643 .insert(label, "machine outage awaiting persistence".into());
2644 }
2645 block_machine_outage(&mut self.state, "implement", &losses);
2648 self.state.save()?;
2649 self.persist_candidates().await?;
2650 for &i in &todo {
2651 self.state
2652 .interrupted_implementations
2653 .insert(self.state.candidates[i].label);
2654 }
2655 return self.stop_for_resource();
2656 }
2657 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2658
2659 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2660 && results
2661 .iter()
2662 .any(|(_, _, out)| FailClass::of(out).is_some());
2663 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2664 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2665 if self
2669 .state
2670 .candidate_preservation
2671 .contains_key(&self.state.candidates[i].label)
2672 {
2673 self.state.seats.insert(seat.key.clone(), seat);
2674 }
2675 continue;
2676 }
2677 let seat_key = seat.key.clone();
2678 let agent = seat.agent.clone();
2687 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2688 self.state.seats.insert(seat.key.clone(), seat);
2689 let label = self.state.candidates[i].label;
2690 self.state.interrupted_implementations.remove(&label);
2691
2692 let (summary, duration, failed, verified_claim) = match out {
2693 AgentOutcome::Ok(o) => {
2694 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2695 let failed = (!o.usable()).then(|| {
2696 if o.timed_out {
2697 "agent timed out".to_owned()
2698 } else {
2699 format!(
2700 "agent exited with {}",
2701 crate::notices::exit_status(o.exit_code)
2702 )
2703 }
2704 });
2705 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2706 (text, o.duration_ms, failed, verified_claim)
2707 }
2708 AgentOutcome::Dropped(o) => {
2714 let why = o
2715 .dropped
2716 .as_ref()
2717 .map(|d| d.why.as_str())
2718 .unwrap_or("the CLI ended the stream without delivering its answer");
2719 (
2720 String::new(),
2721 o.duration_ms,
2722 Some(format!("the CLI dropped the stream ({why})")),
2723 None,
2724 )
2725 }
2726 AgentOutcome::Quota(o) => {
2727 self.state.quota.push(QuotaLoss::from_report(
2728 seat_key,
2729 "implement",
2730 o.quota.as_ref(),
2731 ));
2732 (
2733 String::new(),
2734 o.duration_ms,
2735 Some("rate limited (quota); produced no change".to_owned()),
2736 None,
2737 )
2738 }
2739 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2740 (String::new(), 0, Some(e), None)
2741 }
2742 };
2743
2744 let c = &mut self.state.candidates[i];
2747 if !exhausted_the_fallback_chain {
2748 c.agent = agent;
2749 }
2750 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2751 c.duration_ms = duration;
2752 c.failed = failed;
2753 c.verified_noop = verified_claim;
2754 self.state
2755 .candidate_preservation
2756 .insert(label, "awaiting persistence".to_owned());
2757 }
2758 self.state.save()?;
2759 self.persist_candidates().await?;
2760
2761 if resource_blocked {
2762 return self.stop_for_resource();
2763 }
2764 if selected_failed {
2765 self.state.status = RunStatus::Blocked;
2766 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2767 self.state.save()?;
2768 bail!(
2769 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2770 );
2771 }
2772 self.after_implement()
2773 }
2774
2775 async fn persist_candidates(&mut self) -> Result<()> {
2777 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2778 for label in labels {
2779 let i = self
2780 .state
2781 .candidates
2782 .iter()
2783 .position(|c| c.label == label)
2784 .context("pending candidate is missing")?;
2785 let c = self.state.candidates[i].clone();
2786 let interrupted = self.state.interrupted_implementations.contains(&label);
2787 let result = async {
2788 let wait = crate::repo_lock::WaitPolicy::new(
2789 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2790 || false,
2791 );
2792 let _coordination =
2793 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2794 if let Err(e) = git::snapshot_uncommitted(
2797 &c.worktree,
2798 &self
2799 .state
2800 .dir()
2801 .join("artifacts")
2802 .join(format!("candidate-{label}-recovery")),
2803 )
2804 .await
2805 {
2806 self.state.event(
2807 "preserve",
2808 format!("candidate {label} snapshot incomplete: {e:#}"),
2809 );
2810 }
2811 let rescued = git::rescue_commit(
2812 &c.worktree,
2813 &format!("magi: candidate {label} (uncommitted work)"),
2814 )
2815 .await?;
2816 self.state.note_withheld("implement", &rescued.withheld);
2817 let base = &self.state.base_commit;
2818 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2819 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2820 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2821 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2822 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2823 if c.empty && commits > 0 && !patch.trim().is_empty()
2824 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2825 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2826 || self.state.tally.is_some())
2827 {
2828 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2832 }
2833 let c = &mut self.state.candidates[i];
2834 c.commits = commits;
2835 c.stat = stat;
2836 c.files = files;
2837 let no_changes = commits == 0 || patch.trim().is_empty();
2838 c.empty = no_changes && !interrupted;
2839 if interrupted && no_changes {
2840 c.failed = None;
2841 c.verified_noop = None;
2842 }
2843 if !c.empty {
2844 c.failed = None;
2845 c.verified_noop = None;
2846 }
2847 self.state.event(
2848 "preserve",
2849 format!("candidate {label}: {files} files, {commits} commits"),
2850 );
2851 Ok::<_, anyhow::Error>(())
2852 }
2853 .await;
2854 match result {
2855 Ok(()) => {
2856 self.state.candidate_preservation.remove(&label);
2857 if !interrupted || self.state.candidates[i].files > 0 {
2858 self.state.interrupted_implementations.remove(&label);
2859 }
2860 }
2861 Err(e) => {
2862 let reason = format!("candidate {label} preservation failed: {e:#}");
2863 self.state.event("preserve", &reason);
2864 self.state.candidate_preservation.insert(label, reason);
2865 }
2866 }
2867 if self.state.candidate_preservation.is_empty()
2868 && self.state.status == RunStatus::Blocked
2869 && self.state.candidates.iter().all(|c| c.failed.is_none())
2870 {
2871 self.state.status = RunStatus::Implementing;
2875 self.state.event(
2876 "preserve",
2877 "candidate persistence recovered; implementation can continue",
2878 );
2879 }
2880 self.state.save()?;
2881 }
2882 if !self.state.candidate_preservation.is_empty() {
2883 self.state.status = RunStatus::Blocked;
2884 self.state.save()?;
2885 bail!(
2886 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2887 );
2888 }
2889 Ok(())
2890 }
2891
2892 async fn resume_undelivered(
2920 &mut self,
2921 results: &mut [(usize, SeatState, AgentOutcome)],
2922 sent: &[SeatJob],
2923 prompts: &Prompts,
2924 run_id: &str,
2925 ) {
2926 for (wi, seat, out) in results.iter_mut() {
2927 let Some(dropped) = (match &*out {
2928 AgentOutcome::Dropped(o) => o.dropped.clone(),
2929 _ => None,
2930 }) else {
2931 continue;
2932 };
2933 let Some(job) = sent.get(*wi) else { continue };
2934 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2936 self.state.event(
2937 "implement",
2938 format!(
2939 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2940 work is in the tree",
2941 seat.key, dropped.output_tokens, dropped.why
2942 ),
2943 );
2944 continue;
2945 }
2946 if !has_context(&job.spec, seat, job.sessions) {
2954 self.state.event(
2955 "implement",
2956 format!(
2957 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2958 is no session left to resume",
2959 seat.key, dropped.output_tokens, dropped.why
2960 ),
2961 );
2962 continue;
2963 }
2964 self.state.event(
2965 "implement",
2966 format!(
2967 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2968 conversation",
2969 seat.key, dropped.output_tokens, dropped.why
2970 ),
2971 );
2972 let mut retry = job.clone();
2973 retry.seat = seat.clone();
2974 retry.prompt = prompt::resume_after_drop(&dropped.why);
2975 retry.timeout = retry_budget(job.timeout, true);
2976 retry.stem = format!("{}-resume", job.stem);
2977 let cache = self.state.config.cache_dir();
2978 let ctx = WaveCtx {
2979 carry_seats: false,
2980 pauses: Some((&self.pause, &self.interrupt)),
2981 run: run_id,
2982 node: "implement",
2983 prompts,
2984 cache: cache.as_deref(),
2985 round: None,
2986 };
2987 let (resumed_seat, resumed) =
2988 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2989 *seat = resumed_seat;
2990 *out = resumed;
2991 }
2992 }
2993
2994 async fn resume_seat_handovers(
3059 &mut self,
3060 results: &mut [(usize, SeatState, AgentOutcome)],
3061 sent: &mut [SeatJob],
3062 prompts: &Prompts,
3063 run_id: &str,
3064 ) -> Result<()> {
3065 let instruction = seeded_instruction(&self.state);
3066 let language = self.state.config.graph.language.clone();
3067 let github_language = self.state.config.graph.github_language.clone();
3068 let brief = self
3069 .state
3070 .advice
3071 .as_ref()
3072 .and_then(|a| a.synthesis.as_deref())
3073 .map(str::to_owned);
3074 let attachments = self.state.attachments.clone();
3075 for (wi, seat, out) in results.iter_mut() {
3076 let others: BTreeSet<String> = sent
3079 .iter()
3080 .enumerate()
3081 .filter(|(j, _)| j != wi)
3082 .map(|(_, j)| j.spec.id.clone())
3083 .collect();
3084 let Some(job) = sent.get_mut(*wi) else {
3085 continue;
3086 };
3087 let start = self
3092 .roles
3093 .implementer_roster
3094 .iter()
3095 .position(|s| s.id == job.spec.id)
3096 .unwrap_or(0);
3097 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3098 let mut fallback_attempt = 0usize;
3099 let mut prev: Option<FailClass> = None;
3100 while let Some(cur) = FailClass::of(&*out) {
3101 if !should_hand_over(prev.as_ref(), &cur) {
3102 break;
3103 }
3104 let Some(next) =
3105 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3106 .cloned()
3107 else {
3108 break;
3109 };
3110 tried.insert(next.id.clone());
3111 fallback_attempt += 1;
3112
3113 let coordination = self.coordinate(&job.cwd).await?;
3114 if let Ok(r) = git::rescue_commit(
3115 &job.cwd,
3116 &format!(
3117 "magi: candidate {} (uncommitted work before {} fallback)",
3118 seat.key,
3119 if cur == FailClass::Quota {
3120 "quota"
3121 } else {
3122 "handover"
3123 }
3124 ),
3125 )
3126 .await
3127 {
3128 self.state.note_withheld("implement", &r.withheld);
3129 }
3130 drop(coordination);
3131
3132 record_handover(
3133 &mut self.state,
3134 "implement",
3135 &seat.key,
3136 &seat.agent,
3137 &next.id,
3138 &cur,
3139 &fail_reason(&*out),
3140 );
3141 prev = Some(cur.clone());
3142
3143 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3144 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3145 job.spec = next.clone();
3153 let mut retry = job.clone();
3154 retry.seat = new_seat;
3155 retry.prompt = prompt::implement_in(
3156 &instruction,
3157 &job.cwd.to_string_lossy(),
3158 &language,
3159 &github_language,
3160 brief.as_deref(),
3161 &attachments,
3162 );
3163 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3164 let cache = self.state.config.cache_dir();
3165 let ctx = WaveCtx {
3166 carry_seats: false,
3167 pauses: Some((&self.pause, &self.interrupt)),
3168 run: run_id,
3169 node: "implement",
3170 prompts,
3171 cache: cache.as_deref(),
3172 round: None,
3173 };
3174 let (fallback_seat, fallback_out) = run_one(
3175 retry,
3176 Arc::clone(&self.sem),
3177 &ctx,
3178 &mut self.state,
3179 fallback_attempt,
3180 )
3181 .await;
3182 *seat = fallback_seat;
3183 *out = fallback_out;
3184 }
3185 }
3186 Ok(())
3187 }
3188
3189 async fn resume_unconfirmed_commands(
3213 &mut self,
3214 results: &mut [(usize, SeatState, AgentOutcome)],
3215 sent: &[SeatJob],
3216 prompts: &Prompts,
3217 run_id: &str,
3218 ) {
3219 for (wi, seat, out) in results.iter_mut() {
3220 let AgentOutcome::Ok(o) = &*out else {
3221 continue;
3222 };
3223 if !has_unconfirmed_command(&o.commands) {
3224 continue;
3225 }
3226 let Some(job) = sent.get(*wi) else { continue };
3227 if !has_context(&job.spec, seat, job.sessions) {
3228 self.state.event(
3229 "implement",
3230 format!(
3231 "{}: the reply named a command whose own CLI never confirmed the exit \
3232 status of, but there is no session left to resume",
3233 seat.key
3234 ),
3235 );
3236 continue;
3237 }
3238 self.state.event(
3239 "implement",
3240 format!(
3241 "{}: the reply named a command whose own CLI never confirmed the exit \
3242 status of; resuming the conversation",
3243 seat.key
3244 ),
3245 );
3246 let mut retry = job.clone();
3247 retry.seat = seat.clone();
3248 retry.prompt = prompt::resume_incomplete(
3249 "a command in your last reply had no confirmed exit status",
3250 );
3251 retry.timeout = retry_budget(job.timeout, true);
3252 retry.stem = format!("{}-confirm", job.stem);
3253 let cache = self.state.config.cache_dir();
3254 let ctx = WaveCtx {
3255 carry_seats: false,
3256 pauses: Some((&self.pause, &self.interrupt)),
3257 run: run_id,
3258 node: "implement",
3259 prompts,
3260 cache: cache.as_deref(),
3261 round: None,
3262 };
3263 let (resumed_seat, resumed) =
3264 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3265 *seat = resumed_seat;
3266 *out = resumed;
3267 }
3268 }
3269
3270 async fn continue_fix_report(
3291 &mut self,
3292 mut seat: SeatState,
3293 parse_err: String,
3294 job: &SeatJob,
3295 prompts: &Prompts,
3296 run_id: &str,
3297 round: usize,
3298 ) -> (
3299 SeatState,
3300 Option<FixReport>,
3301 Option<String>,
3302 ContinuationRecord,
3303 ) {
3304 let mut last_err = parse_err;
3305 let mut cumulative_wait_ms = 0u64;
3306 let mut attempts = 0usize;
3307 loop {
3308 if !has_context(&job.spec, &seat, job.sessions) {
3309 self.state.event(
3310 "fix",
3311 format!(
3312 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3313 session left to resume into"
3314 ),
3315 );
3316 let outcome = if attempts == 0 {
3317 ContinuationOutcome::NoSession
3318 } else {
3319 ContinuationOutcome::Exhausted
3320 };
3321 return (
3322 seat,
3323 None,
3324 Some(format!("unparsable fix report: {last_err}")),
3325 ContinuationRecord {
3326 attempts,
3327 cumulative_wait_ms,
3328 outcome,
3329 },
3330 );
3331 }
3332 if attempts >= MAX_FIX_CONTINUATIONS {
3333 self.state.event(
3334 "fix",
3335 format!(
3336 "round {round}: fixer's reply still had no adoption report after \
3337 {attempts} continuation(s) ({last_err}); giving up"
3338 ),
3339 );
3340 return (
3341 seat,
3342 None,
3343 Some(format!(
3344 "unparsable fix report after {attempts} continuation(s): {last_err}"
3345 )),
3346 ContinuationRecord {
3347 attempts,
3348 cumulative_wait_ms,
3349 outcome: ContinuationOutcome::Exhausted,
3350 },
3351 );
3352 }
3353 attempts += 1;
3354 self.state.event(
3355 "fix",
3356 format!(
3357 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3358 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3359 ),
3360 );
3361 let mut retry = job.clone();
3362 retry.seat = seat.clone();
3363 retry.prompt = prompt::resume_incomplete(&last_err);
3364 retry.timeout = retry_budget(job.timeout, true);
3365 retry.stem = format!("{}-continue{attempts}", job.stem);
3366 let cache = self.state.config.cache_dir();
3367 let ctx = WaveCtx {
3368 carry_seats: false,
3369 pauses: Some((&self.pause, &self.interrupt)),
3370 run: run_id,
3371 node: "fix",
3372 prompts,
3373 cache: cache.as_deref(),
3374 round: Some(round),
3375 };
3376 let (resumed_seat, resumed_out) = run_one(
3377 retry,
3378 Arc::clone(&self.sem),
3379 &ctx,
3380 &mut self.state,
3381 attempts,
3382 )
3383 .await;
3384 seat = resumed_seat;
3385 match resumed_out {
3386 AgentOutcome::Ok(o) => {
3387 cumulative_wait_ms += o.duration_ms;
3388 match verdict::extract_json::<FixReport>(&o.text) {
3389 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3390 self.state.event(
3391 "fix",
3392 format!(
3393 "round {round}: fixer's adoption report recovered after \
3394 {attempts} continuation(s)"
3395 ),
3396 );
3397 return (
3398 seat,
3399 Some(report),
3400 None,
3401 ContinuationRecord {
3402 attempts,
3403 cumulative_wait_ms,
3404 outcome: ContinuationOutcome::Resumed,
3405 },
3406 );
3407 }
3408 Ok(_) => {
3416 last_err = "the reply parsed, but it reported a command whose own CLI \
3417 never confirmed an exit status"
3418 .to_owned();
3419 }
3420 Err(e) => last_err = e.to_string(),
3421 }
3422 }
3423 AgentOutcome::Quota(o) => {
3424 cumulative_wait_ms += o.duration_ms;
3425 let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3426 block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3427 self.state.quota.push(loss);
3428 self.state.event(
3429 "fix",
3430 format!(
3431 "round {round}: continuation rate limited (quota); not retrying now"
3432 ),
3433 );
3434 return (
3435 seat,
3436 None,
3437 Some("rate limited (quota) while recovering the fix report".to_owned()),
3438 ContinuationRecord {
3439 attempts,
3440 cumulative_wait_ms,
3441 outcome: ContinuationOutcome::QuotaLost,
3442 },
3443 );
3444 }
3445 AgentOutcome::Dropped(o) => {
3446 cumulative_wait_ms += o.duration_ms;
3447 let why = o
3448 .dropped
3449 .as_ref()
3450 .map(|d| d.why.as_str())
3451 .unwrap_or("the CLI ended the stream without delivering its answer");
3452 last_err = format!("the CLI dropped the stream ({why})");
3453 }
3454 AgentOutcome::Failed(e) => last_err = e,
3455 AgentOutcome::ResourceBlocked(e) => {
3458 return (
3459 seat,
3460 None,
3461 Some(format!("the continuation was not started: {e}")),
3462 ContinuationRecord {
3463 attempts: attempts.saturating_sub(1),
3464 cumulative_wait_ms,
3465 outcome: ContinuationOutcome::NoSession,
3466 },
3467 );
3468 }
3469 }
3470 }
3471 }
3472
3473 fn after_implement(&mut self) -> Result<()> {
3474 if !self.state.candidate_preservation.is_empty() {
3475 bail!("candidate preservation incomplete; judging is unsafe");
3476 }
3477 if self.state.leaks.is_empty() {
3479 let cfg = self.state.config.blind.clone();
3480 let mut leaks = Vec::new();
3481 for c in &self.state.candidates {
3482 let Some(patch) =
3483 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3484 else {
3485 continue;
3486 };
3487 leaks.extend(blind::scan(
3488 &format!("candidate {} patch", c.label),
3489 &patch,
3490 &cfg.vendor_tokens,
3491 ));
3492 }
3493 if !leaks.is_empty() {
3494 let summary = leaks
3495 .iter()
3496 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3497 .collect::<Vec<_>>()
3498 .join(", ");
3499 match cfg.on_leak {
3500 LeakPolicy::Fail => {
3501 self.state.status = RunStatus::Failed;
3502 self.state
3503 .event("blind", format!("vendor text in a patch: {summary}"));
3504 self.state.leaks = leaks;
3505 self.state.save()?;
3506 self.settle_questions();
3507 bail!(
3508 "blind.on_leak = \"fail\" and vendor text reached a \
3509 judged patch: {summary}"
3510 );
3511 }
3512 LeakPolicy::Redact => self.state.event(
3513 "blind",
3514 format!("redacting vendor text for judging: {summary}"),
3515 ),
3516 LeakPolicy::Warn => self.state.event(
3517 "blind",
3518 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3519 ),
3520 }
3521 self.state.leaks = leaks;
3522 }
3523 }
3524
3525 if self.state.viable().is_empty() {
3526 if self.state.all_candidates_verified_noop() {
3527 self.state.status = RunStatus::VerifiedNoop;
3538 self.state.save()?;
3539 self.settle_questions();
3540 return Ok(());
3541 }
3542 self.state.status = RunStatus::Failed;
3543 self.state.save()?;
3544 self.settle_questions();
3545 bail!("no candidate produced a change; nothing to judge");
3546 }
3547 self.state.status = RunStatus::Judging;
3548 self.state.save()?;
3549 Ok(())
3550 }
3551
3552 async fn judge(&mut self) -> Result<()> {
3555 let run_id = self.state.id.clone();
3560 let prompts = self.state.config.prompts.clone();
3561 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3562 return Ok(());
3563 }
3564 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3565 if viable.len() == 1 {
3566 self.state.judge_skipped = true;
3573 self.state.event(
3574 "judge",
3575 format!(
3576 "only candidate {} produced a change; judging skipped",
3577 viable[0].label
3578 ),
3579 );
3580 self.state.save()?;
3581 return Ok(());
3582 }
3583 self.state.status = RunStatus::Judging;
3584
3585 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3586 let language = self.state.config.graph.language.clone();
3587 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3588 let sessions = self.state.config.graph.sessions;
3589 let artifacts = agent::artifacts_dir(&self.state.dir());
3590 let root = self.state.worktree_root();
3591 let base_short = short(&self.state.base_commit);
3592
3593 let mut jobs = Vec::new();
3594 let mut orders = Vec::new();
3595 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3596 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3597 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3598 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3599 let seat_key = format!("judge-{}", j + 1);
3600 let seat = self.seat(&seat_key, &spec.id);
3601 jobs.push(SeatJob {
3602 prompt: prompt::judge(
3603 &self.state.instruction,
3604 &views,
3605 self.roles.judges.len(),
3606 &base_short,
3607 &language,
3608 ),
3609 spec,
3610 seat,
3611 cwd: root.join(format!("judge-{}", j + 1)),
3612 timeout,
3613 allow_write: false,
3614 sessions,
3615 artifacts: artifacts.clone(),
3616 stem: format!("judge-{}", j + 1),
3617 handover: None,
3618 });
3619 }
3620
3621 self.state.event(
3622 "judge",
3623 format!(
3624 "{} judges ranking {} candidates blind",
3625 jobs.len(),
3626 viable.len()
3627 ),
3628 );
3629 let labels_for_check = labels.clone();
3630 let mut quota_losses = Vec::new();
3631 let cache = self.state.config.cache_dir();
3632 let ctx = WaveCtx {
3633 carry_seats: false,
3634 pauses: Some((&self.pause, &self.interrupt)),
3635 run: &run_id,
3636 node: "judge",
3637 prompts: &prompts,
3638 cache: cache.as_deref(),
3639 round: None,
3640 };
3641 let results = ask_json_wave::<Ranking>(
3642 jobs,
3643 Arc::clone(&self.sem),
3644 self.state.config.graph.retries,
3645 &self.roles.judge_roster,
3646 &ctx,
3647 &mut quota_losses,
3648 &mut self.state,
3649 &move |r: &Ranking| r.validate(&labels_for_check),
3650 )
3651 .await;
3652 if self.state.resource_block.is_some() {
3653 return self.stop_for_resource();
3654 }
3655 self.state.quota.extend(quota_losses);
3656
3657 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3658 let agent_id = seat.agent.clone();
3659 self.state.seats.insert(seat.key.clone(), seat);
3660 let mut record = Judgement {
3661 judge: j + 1,
3662 seat: format!("judge-{}", j + 1),
3663 agent: agent_id,
3664 ranking: Vec::new(),
3665 reasons: BTreeMap::new(),
3666 confidence: None,
3667 strengths: BTreeMap::new(),
3668 order: orders[j].clone(),
3669 failed: None,
3670 duration_ms: 0,
3671 };
3672 match res {
3673 Ok((ranking, out)) => {
3674 record.ranking = ranking.normalized();
3675 record.reasons = ranking.reasons;
3676 record.strengths = ranking.strengths;
3677 record.confidence = ranking.confidence;
3678 record.duration_ms = out.duration_ms;
3679 self.state.event(
3680 "judge",
3681 format!(
3682 "judge {} ranked {}",
3683 j + 1,
3684 record.ranking.iter().collect::<String>()
3685 ),
3686 );
3687 }
3688 Err(e) => {
3689 record.failed = Some(e.to_string());
3690 self.state
3691 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3692 }
3693 }
3694 self.state.judgements.push(record);
3695 self.state.save()?;
3696 }
3697 Ok(())
3698 }
3699
3700 async fn deliberate(&mut self) -> Result<()> {
3703 if self.hold_base_sync().await? {
3704 return Ok(());
3705 }
3706 let run_id = self.state.id.clone();
3711 let prompts = self.state.config.prompts.clone();
3712 let rounds = self.state.config.graph.deliberate_rounds;
3713 if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
3715 return Ok(());
3716 }
3717 let tops: Vec<char> = self
3718 .state
3719 .judgements
3720 .iter()
3721 .filter_map(|j| j.ranking.first().copied())
3722 .collect();
3723 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3724 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3725 self.state.event(
3726 "deliberate",
3727 format!("judges agreed on {} outright; no deliberation", tops[0]),
3728 );
3729 }
3730 self.state.status = RunStatus::Voting;
3731 self.state.save()?;
3732 return Ok(());
3733 }
3734
3735 self.state.status = RunStatus::Deliberating;
3736 self.state.event(
3737 "deliberate",
3738 format!(
3739 "split: first choices were {} — opening {rounds} round(s)",
3740 tops.iter().collect::<String>()
3741 ),
3742 );
3743
3744 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3745 let language = self.state.config.graph.language.clone();
3746 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3747 let sessions = self.state.config.graph.sessions;
3748 let artifacts = agent::artifacts_dir(&self.state.dir());
3749 let root = self.state.worktree_root();
3750 let base_short = short(&self.state.base_commit);
3751
3752 for round in self.state.deliberation.len() + 1..=rounds {
3756 let mut turns: Vec<DeliberationTurn> = Vec::new();
3757 let mut round_losses = Vec::new();
3758 let mut asked = 0;
3759 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3760 if self.state.judgements[j].failed.is_some() {
3761 continue;
3762 }
3763 let seat_key = format!("judge-{}", j + 1);
3764 let spec = self.occupant(&seat_key, spec);
3765 let mut seat = self.seat(&seat_key, &spec.id);
3766 let transcript = self.transcript(&turns, j);
3767 let build = |context: Option<&str>| {
3768 prompt::deliberate(
3769 &self.state.instruction,
3770 context,
3771 &transcript,
3772 round,
3773 rounds,
3774 &language,
3775 )
3776 };
3777 let block = self.candidate_block(&viable, &base_short);
3778 let full = build(Some(&block));
3779 let text = if has_context(&spec, &seat, sessions) {
3780 build(None)
3781 } else {
3782 full.clone()
3783 };
3784 let job = SeatJob {
3785 spec,
3786 seat: seat.clone(),
3787 prompt: text,
3788 cwd: root.join(format!("judge-{}", j + 1)),
3789 timeout,
3790 allow_write: false,
3791 sessions,
3792 artifacts: artifacts.clone(),
3793 stem: format!("delib-{round}-judge-{}", j + 1),
3794 handover: Some(full),
3795 };
3796 let cache = self.state.config.cache_dir();
3797 let ctx = WaveCtx {
3798 carry_seats: false,
3799 pauses: Some((&self.pause, &self.interrupt)),
3800 run: &run_id,
3801 node: "deliberate",
3802 prompts: &prompts,
3803 cache: cache.as_deref(),
3804 round: None,
3805 };
3806 let mut losses = Vec::new();
3810 asked += 1;
3811 let mut results = ask_wave_with::<String>(
3812 vec![job],
3813 Arc::clone(&self.sem),
3814 0,
3815 &self.roles.judge_roster,
3816 &ctx,
3817 &mut losses,
3818 &mut self.state,
3819 &|text: &str| {
3820 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3821 },
3822 )
3823 .await;
3824 if self.state.resource_block.is_some() {
3825 return self.stop_for_resource();
3826 }
3827 round_losses.extend(losses.iter().cloned());
3828 self.state.quota.extend(losses);
3829 let (updated, res, _) = results.pop().expect("one job in, one result out");
3830 seat = updated;
3831 let agent_id = seat.agent.clone();
3832 self.state.seats.insert(seat.key.clone(), seat);
3833 let body = match res {
3834 Ok((body, _)) => body,
3835 Err(e) => {
3838 self.state
3839 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3840 continue;
3841 }
3842 };
3843 let tentative = verdict::extract_json::<Position>(&body)
3844 .ok()
3845 .and_then(|p| p.tentative)
3846 .and_then(|s| s.trim().chars().next())
3847 .map(|c| c.to_ascii_uppercase());
3848 self.state.event(
3849 "deliberate",
3850 format!(
3851 "round {round}: judge {} now favours {}",
3852 j + 1,
3853 tentative.map_or("—".to_owned(), |c| c.to_string())
3854 ),
3855 );
3856 turns.push(DeliberationTurn {
3857 judge: j + 1,
3858 agent: agent_id,
3859 body: blind::sanitize_prose(&body, &self.state.config.blind),
3860 tentative,
3861 });
3862 }
3863 if asked > 0 && turns.is_empty() && round_losses.len() == asked {
3864 block_machine_outage(&mut self.state, "deliberate", &round_losses);
3865 return self.stop_for_resource();
3866 }
3867 self.state
3868 .deliberation
3869 .push(DeliberationRound { round, turns });
3870 self.state.save()?;
3871 }
3872
3873 self.state.status = RunStatus::Voting;
3874 self.state.save()?;
3875 Ok(())
3876 }
3877
3878 async fn vote(&mut self) -> Result<()> {
3881 if self.hold_base_sync().await? {
3882 return Ok(());
3883 }
3884 let run_id = self.state.id.clone();
3889 let prompts = self.state.config.prompts.clone();
3890 if !self.state.votes.is_empty() || self.state.tally.is_some() {
3891 return Ok(());
3892 }
3893 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3894 if viable.len() == 1 {
3895 return Ok(());
3896 }
3897 self.state.status = RunStatus::Voting;
3898
3899 let language = self.state.config.graph.language.clone();
3900 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3901 let sessions = self.state.config.graph.sessions;
3902 let artifacts = agent::artifacts_dir(&self.state.dir());
3903 let root = self.state.worktree_root();
3904 let base_short = short(&self.state.base_commit);
3905 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3906
3907 let mut jobs = Vec::new();
3908 let mut seats_at = Vec::new();
3909 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3910 if self
3911 .state
3912 .judgements
3913 .get(j)
3914 .is_some_and(|r| r.failed.is_some())
3915 {
3916 continue;
3917 }
3918 let seat_key = format!("judge-{}", j + 1);
3919 let spec = self.occupant(&seat_key, spec);
3920 let seat = self.seat(&seat_key, &spec.id);
3921 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3922 let text = if has_context(&spec, &seat, sessions) {
3923 prompt::final_vote(&viable, &language)
3924 } else {
3925 full.clone()
3926 };
3927 jobs.push(SeatJob {
3928 spec,
3929 seat,
3930 prompt: text,
3931 cwd: root.join(format!("judge-{}", j + 1)),
3932 timeout,
3933 allow_write: false,
3934 sessions,
3935 artifacts: artifacts.clone(),
3936 stem: format!("vote-judge-{}", j + 1),
3937 handover: Some(full),
3938 });
3939 seats_at.push(j);
3940 }
3941
3942 self.state.event(
3943 "vote",
3944 format!(
3945 "collecting {} final votes one by one, privately",
3946 jobs.len()
3947 ),
3948 );
3949 let allowed = viable.clone();
3950 let mut quota_losses = Vec::new();
3951 let cache = self.state.config.cache_dir();
3952 let ctx = WaveCtx {
3953 carry_seats: false,
3954 pauses: Some((&self.pause, &self.interrupt)),
3955 run: &run_id,
3956 node: "vote",
3957 prompts: &prompts,
3958 cache: cache.as_deref(),
3959 round: None,
3960 };
3961 let results = ask_json_wave::<FinalVote>(
3962 jobs,
3963 Arc::clone(&self.sem),
3964 self.state.config.graph.retries,
3965 &self.roles.judge_roster,
3966 &ctx,
3967 &mut quota_losses,
3968 &mut self.state,
3969 &move |v: &FinalVote| match v.label() {
3970 Some(c) if allowed.contains(&c) => Ok(()),
3971 other => bail!("vote {other:?} is not one of {allowed:?}"),
3972 },
3973 )
3974 .await;
3975 if self.state.resource_block.is_some() {
3976 return self.stop_for_resource();
3977 }
3978 self.state.quota.extend(quota_losses);
3979
3980 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3981 let agent_id = seat.agent.clone();
3982 self.state.seats.insert(seat.key.clone(), seat);
3983 let initial = self
3984 .state
3985 .judgements
3986 .get(j)
3987 .and_then(|r| r.ranking.first().copied());
3988 let mut record = VoteRecord {
3989 judge: j + 1,
3990 agent: agent_id,
3991 vote: None,
3992 reason: String::new(),
3993 changed: false,
3994 };
3995 match res {
3996 Ok((v, _)) => {
3997 record.vote = v.label();
3998 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3999 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4000 self.state.event(
4001 "vote",
4002 format!(
4003 "judge {} voted {}{}",
4004 j + 1,
4005 record.vote.unwrap_or('?'),
4006 if record.changed { " (changed)" } else { "" }
4007 ),
4008 );
4009 }
4010 Err(e) => {
4011 self.state
4012 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4013 }
4014 }
4015 self.state.votes.push(record);
4016 self.state.save()?;
4017 }
4018 Ok(())
4019 }
4020
4021 fn tally(&mut self) -> Result<()> {
4024 if self.state.tally.is_some() {
4025 return Ok(());
4026 }
4027 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4028 let tops: Vec<char> = self
4029 .state
4030 .judgements
4031 .iter()
4032 .filter_map(|j| j.ranking.first().copied())
4033 .collect();
4034 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4035
4036 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4039 let mut cast: Vec<char> = Vec::new();
4040 for (i, j) in self.state.judgements.iter().enumerate() {
4041 let vote = self
4042 .state
4043 .votes
4044 .iter()
4045 .find(|v| v.judge == i + 1)
4046 .and_then(|v| v.vote)
4047 .or_else(|| j.ranking.first().copied());
4048 if let Some(v) = vote {
4049 *first_choice.entry(v).or_insert(0) += 1;
4050 cast.push(v);
4051 }
4052 }
4053
4054 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4055 for j in &self.state.judgements {
4056 let n = j.ranking.len();
4057 for (pos, label) in j.ranking.iter().enumerate() {
4058 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4059 }
4060 }
4061
4062 let best = first_choice.values().copied().max().unwrap_or(0);
4063 let mut leaders: Vec<char> = first_choice
4064 .iter()
4065 .filter(|(_, v)| **v == best)
4066 .map(|(k, _)| *k)
4067 .collect();
4068 let mut tie_break = None;
4069 if leaders.len() > 1 {
4070 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4071 let borda_leaders: Vec<char> = leaders
4072 .iter()
4073 .copied()
4074 .filter(|l| borda[l] == top_borda)
4075 .collect();
4076 tie_break = Some(if borda_leaders.len() == 1 {
4077 format!(
4078 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4079 leaders.len()
4080 )
4081 } else {
4082 format!(
4083 "{} way tie on both first-choice votes and Borda points, broken by label order",
4084 leaders.len()
4085 )
4086 });
4087 leaders = borda_leaders;
4088 leaders.sort_unstable();
4089 }
4090 let winner = *leaders
4091 .first()
4092 .or(viable.first())
4093 .context("no candidate to declare a winner from")?;
4094
4095 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4096 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4097 let deliberated = !self.state.deliberation.is_empty();
4098
4099 let quota_seats: std::collections::BTreeSet<&str> =
4103 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4104 let mut present = 0usize;
4105 for (i, j) in self.state.judgements.iter().enumerate() {
4106 if quota_seats.contains(j.seat.as_str()) {
4107 continue;
4108 }
4109 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4110 let voted = self
4111 .state
4112 .votes
4113 .iter()
4114 .any(|v| v.judge == i + 1 && v.vote.is_some());
4115 if ranked || voted {
4116 present += 1;
4117 }
4118 }
4119 let needs_quorum = viable.len() > 1;
4125 let judges_total = if needs_quorum {
4126 self.roles.judges.len()
4127 } else {
4128 0
4129 };
4130 let quorum = if needs_quorum {
4131 judges_total / 2 + 1
4132 } else {
4133 0
4134 };
4135 let met_quorum = !needs_quorum || present >= quorum;
4136 let uncontested = (!needs_quorum).then(|| {
4137 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4138 });
4139
4140 self.state.event(
4141 "tally",
4142 match &uncontested {
4143 Some(reason) => format!("winner {winner} — {reason}"),
4144 None => format!(
4145 "winner {winner} — votes {} | initial {} | {} changed | \
4146 {present}/{judges_total} judges{}",
4147 first_choice
4148 .iter()
4149 .map(|(k, v)| format!("{k}:{v}"))
4150 .collect::<Vec<_>>()
4151 .join(" "),
4152 if unanimous_initial {
4153 "unanimous"
4154 } else {
4155 "split"
4156 },
4157 changed_votes,
4158 if met_quorum {
4159 String::new()
4160 } else {
4161 format!(" — below quorum ({quorum} required)")
4162 },
4163 ),
4164 },
4165 );
4166 if !met_quorum {
4167 self.state.event(
4168 "stall",
4169 format!(
4170 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4171 the run stops here, resumable"
4172 ),
4173 );
4174 }
4175 self.state.tally = Some(Tally {
4176 first_choice,
4177 borda,
4178 winner,
4179 rankings: tops.len(),
4180 unanimous_initial,
4181 deliberated,
4182 changed_votes,
4183 unanimous_final,
4184 tie_break,
4185 judges: judges_total,
4186 present,
4187 quorum,
4188 met_quorum,
4189 uncontested,
4190 });
4191 self.state.status = if met_quorum {
4192 RunStatus::Reviewing
4193 } else {
4194 RunStatus::Stalled
4195 };
4196 self.state.save()?;
4197 Ok(())
4198 }
4199
4200 #[allow(clippy::too_many_lines)]
4221 async fn recover_stall(&mut self) -> Result<bool> {
4222 let run_id = self.state.id.clone();
4227 let prompts = self.state.config.prompts.clone();
4228 let quota_seats: BTreeSet<&str> =
4233 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4234 let absent: Vec<String> = self
4235 .state
4236 .judgements
4237 .iter()
4238 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4239 .map(|j| j.seat.clone())
4240 .collect();
4241 if absent.is_empty() {
4242 return Ok(false);
4243 }
4244 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4245 if viable.len() <= 1 {
4246 return Ok(false);
4247 }
4248 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4249 let language = self.state.config.graph.language.clone();
4250 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4251 let sessions = self.state.config.graph.sessions;
4252 let artifacts = agent::artifacts_dir(&self.state.dir());
4253 let root = self.state.worktree_root();
4254 let base_short = short(&self.state.base_commit);
4255 let candidates: Vec<Candidate> = viable.clone();
4256
4257 let mut positions: Vec<usize> = absent
4259 .iter()
4260 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4261 .collect();
4262 if positions.is_empty() {
4263 return Ok(false);
4264 }
4265 positions.sort_unstable();
4266 positions.dedup();
4267
4268 let mut judge_jobs = Vec::new();
4270 for &j in &positions {
4271 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4272 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4273 let seat_key = format!("judge-{}", j + 1);
4274 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4275 let seat = self.seat(&seat_key, &spec.id);
4276 judge_jobs.push(SeatJob {
4277 spec,
4278 seat,
4279 prompt: prompt::judge(
4280 &self.state.instruction,
4281 &views,
4282 self.roles.judges.len(),
4283 &base_short,
4284 &language,
4285 ),
4286 cwd: root.join(seat_key),
4287 timeout,
4288 allow_write: false,
4289 sessions,
4290 artifacts: artifacts.clone(),
4291 stem: format!("judge-{}-recover", j + 1),
4292 handover: None,
4293 });
4294 }
4295
4296 let labels_for_check = labels.clone();
4297 let mut judge_losses = Vec::new();
4298 let retries = self.state.config.graph.retries;
4299 let cache = self.state.config.cache_dir();
4300 let ctx = WaveCtx {
4301 carry_seats: false,
4302 pauses: Some((&self.pause, &self.interrupt)),
4303 run: &run_id,
4304 node: "judge",
4305 prompts: &prompts,
4306 cache: cache.as_deref(),
4307 round: None,
4308 };
4309 let results = ask_json_wave::<Ranking>(
4310 judge_jobs,
4311 Arc::clone(&self.sem),
4312 retries,
4313 &self.roles.judge_roster,
4314 &ctx,
4315 &mut judge_losses,
4316 &mut self.state,
4317 &move |r: &Ranking| r.validate(&labels_for_check),
4318 )
4319 .await;
4320 if self.state.resource_block.is_some() {
4321 self.stop_for_resource()?;
4322 return Ok(false);
4323 }
4324
4325 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4327 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4328 let agent_id = seat.agent.clone();
4329 self.state.seats.insert(seat.key.clone(), seat);
4330 let record = &mut self.state.judgements[j];
4331 match res {
4332 Ok((ranking, out)) => {
4333 record.agent = agent_id;
4334 record.ranking = ranking.normalized();
4335 record.reasons = ranking.reasons;
4336 record.strengths = ranking.strengths;
4337 record.confidence = ranking.confidence;
4338 record.failed = None;
4339 record.duration_ms = out.duration_ms;
4340 recovered.insert(j);
4341 self.state.event(
4342 "recover",
4343 format!("judge {} ranked again after the limit", j + 1),
4344 );
4345 }
4346 Err(e) => {
4347 self.state
4348 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4349 }
4350 }
4351 }
4352
4353 let mut vote_jobs = Vec::new();
4355 let mut vote_pos: Vec<usize> = Vec::new();
4356 for &j in &recovered {
4357 let seat_key = format!("judge-{}", j + 1);
4358 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4359 let seat = self.seat(&seat_key, &spec.id);
4360 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4361 let text = if has_context(&spec, &seat, sessions) {
4362 prompt::final_vote(&labels, &language)
4363 } else {
4364 full.clone()
4365 };
4366 vote_jobs.push(SeatJob {
4367 spec,
4368 seat,
4369 prompt: text,
4370 cwd: root.join(seat_key),
4371 timeout,
4372 allow_write: false,
4373 sessions,
4374 artifacts: artifacts.clone(),
4375 stem: format!("vote-judge-{}-recover", j + 1),
4376 handover: Some(full),
4377 });
4378 vote_pos.push(j);
4379 }
4380 let allowed = labels.clone();
4381 let mut vote_losses = Vec::new();
4382 let vote_retries = self.state.config.graph.retries;
4383 let vote_cache = self.state.config.cache_dir();
4384 let ctx = WaveCtx {
4385 carry_seats: false,
4386 pauses: Some((&self.pause, &self.interrupt)),
4387 run: &run_id,
4388 node: "vote",
4389 prompts: &prompts,
4390 cache: vote_cache.as_deref(),
4391 round: None,
4392 };
4393 let votes = ask_json_wave::<FinalVote>(
4394 vote_jobs,
4395 Arc::clone(&self.sem),
4396 vote_retries,
4397 &self.roles.judge_roster,
4398 &ctx,
4399 &mut vote_losses,
4400 &mut self.state,
4401 &move |v: &FinalVote| match v.label() {
4402 Some(c) if allowed.contains(&c) => Ok(()),
4403 other => bail!("vote {other:?} is not one of {allowed:?}"),
4404 },
4405 )
4406 .await;
4407 if self.state.resource_block.is_some() {
4408 self.stop_for_resource()?;
4409 return Ok(false);
4410 }
4411 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4412 let agent_id = seat.agent.clone();
4413 self.state.seats.insert(seat.key.clone(), seat);
4414 match res {
4415 Ok((v, _)) => {
4416 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4417 rec.vote = v.label();
4418 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4419 } else {
4420 self.state.votes.push(VoteRecord {
4421 judge: j + 1,
4422 agent: agent_id,
4423 vote: v.label(),
4424 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4425 changed: false,
4426 });
4427 }
4428 self.state.event(
4429 "recover",
4430 format!("judge {} voted again after the limit", j + 1),
4431 );
4432 }
4433 Err(e) => {
4434 self.state
4435 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4436 }
4437 }
4438 }
4439
4440 let recovered_keys: BTreeSet<String> = recovered
4444 .iter()
4445 .map(|&j| format!("judge-{}", j + 1))
4446 .collect();
4447 self.state
4448 .quota
4449 .retain(|q| !recovered_keys.contains(&q.seat));
4450 for loss in judge_losses.into_iter().chain(vote_losses) {
4454 if recovered_keys.contains(&loss.seat) {
4455 continue;
4456 }
4457 self.state.quota.retain(|q| q.seat != loss.seat);
4458 self.state.quota.push(loss);
4459 }
4460
4461 self.state.tally = None;
4463 self.tally()?;
4464 Ok(self
4465 .state
4466 .tally
4467 .as_ref()
4468 .map(|t| t.met_quorum)
4469 .unwrap_or(false))
4470 }
4471
4472 async fn graft(&mut self) -> Result<()> {
4488 let Err(e) = self.graft_inner().await else {
4489 return Ok(());
4490 };
4491 if e.is::<crate::repo_lock::ResourceBlocked>() {
4492 return Err(e);
4493 }
4494 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4495 self.state.status = RunStatus::Blocked;
4496 self.state.event("graft", format!("blocked: {e:#}"));
4497 self.state.save()?;
4498 return Err(e);
4499 }
4500 if let Some(rec) = self.state.graft.clone()
4505 && !rec.outcome.settled()
4506 {
4507 if let Some(winner) = self.state.winner().cloned() {
4508 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4509 if re.is::<crate::repo_lock::ResourceBlocked>() {
4510 return Err(re);
4511 }
4512 self.state.status = RunStatus::Blocked;
4513 self.state.event("graft", format!("blocked: {re:#}"));
4514 self.state.save()?;
4515 return Err(re);
4516 }
4517 }
4518 if let Some(g) = self.state.graft.as_mut() {
4519 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4520 }
4521 }
4522 self.state.event("graft", format!("skipped: {e:#}"));
4523 if let Err(e) = self.state.save() {
4524 tracing::warn!("save after graft: {e:#}");
4525 }
4526 Ok(())
4527 }
4528
4529 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4532 let _coordination = self.coordinate(wt).await?;
4533 crate::recovery::restore_repair(wt, pre)
4534 .await
4535 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4536 }
4537
4538 fn graft_skipped(&mut self, why: &str) {
4540 let message = format!("skipped: {why}");
4541 if !self
4542 .state
4543 .events
4544 .iter()
4545 .any(|e| e.node == "graft" && e.message == message)
4546 {
4547 self.state.event("graft", message);
4548 }
4549 }
4550
4551 async fn graft_inner(&mut self) -> Result<()> {
4552 if let Some(rec) = self.state.graft.clone()
4555 && !rec.outcome.settled()
4556 {
4557 if self.state.reviews.is_empty()
4561 && let Some(winner) = self.state.winner().cloned()
4562 {
4563 self.graft_restore(&winner.worktree, &rec.pre).await?;
4564 }
4565 if rec.outcome == GraftOutcome::WaitingForRestore {
4566 self.state.graft = None;
4567 self.state.save()?;
4568 } else {
4569 if let Some(g) = self.state.graft.as_mut() {
4570 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4571 }
4572 self.state.event(
4573 "graft",
4574 "rolled back: the previous round was interrupted; the winner is untouched",
4575 );
4576 self.state.save()?;
4577 return Ok(());
4578 }
4579 }
4580 let plan = match graft::plan(&self.state) {
4581 Ok(plan) => plan,
4582 Err(skip) => {
4583 if skip != graft::Skip::AlreadyDone {
4584 self.graft_skipped(skip.reason());
4585 self.state.save()?;
4586 }
4587 return Ok(());
4588 }
4589 };
4590 let Some(winner) = self.state.winner().cloned() else {
4591 return Ok(());
4592 };
4593 let wt = winner.worktree.clone();
4594 if !git::is_clean(&wt).await? {
4595 self.graft_skipped("the winner's worktree has uncommitted changes");
4596 self.state.save()?;
4597 return Ok(());
4598 }
4599 let pre = git::rev_parse(&wt, "HEAD").await?;
4600 self.state.graft = Some(GraftRecord {
4603 pre: pre.clone(),
4604 post: None,
4605 lines: None,
4606 limit: None,
4607 agent: None,
4608 outcome: GraftOutcome::Running,
4609 requested: graft::requested(&plan),
4610 adoption: None,
4611 adoption_error: None,
4612 });
4613 self.state.event(
4614 "graft",
4615 format!(
4616 "porting strengths of candidate(s) {} onto {}",
4617 plan.strengths
4618 .iter()
4619 .map(|(l, _, _)| l.to_string())
4620 .collect::<Vec<_>>()
4621 .join(", "),
4622 plan.winner
4623 ),
4624 );
4625 self.state.save()?;
4626
4627 let fix_prompt = prompt::graft(
4628 &self.state.instruction,
4629 plan.winner,
4630 &plan.strengths,
4631 &self.state.config.graph.language,
4632 );
4633 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4634 let sessions = self.state.config.graph.sessions;
4635 let artifacts = agent::artifacts_dir(&self.state.dir());
4636 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4637 let (_, seat, out) = self
4638 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4639 SeatJob {
4640 prompt: fix_prompt.clone(),
4641 spec,
4642 seat,
4643 cwd: wt.clone(),
4644 timeout,
4645 allow_write: true,
4646 sessions,
4647 artifacts: artifacts.clone(),
4648 stem: "graft".to_owned(),
4649 handover: None,
4650 }
4651 })
4652 .await;
4653 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4654 self.state.graft.as_mut().expect("graft record").outcome =
4657 GraftOutcome::WaitingForRestore;
4658 return self.stop_for_resource();
4659 }
4660 let mut why: Option<String> = None;
4661 match &out {
4662 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4663 why = Some("the fixer produced nothing usable".to_owned());
4664 }
4665 AgentOutcome::Ok(_) => {}
4666 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4667 AgentOutcome::Quota(o) => {
4668 self.state.quota.push(QuotaLoss::from_report(
4669 seat.key.clone(),
4670 "graft",
4671 o.quota.as_ref(),
4672 ));
4673 why = Some("rate limited (quota)".to_owned());
4674 }
4675 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4676 why = Some(format!(
4677 "the fixer failed: {}",
4678 e.lines()
4679 .next()
4680 .unwrap_or("")
4681 .chars()
4682 .take(160)
4683 .collect::<String>()
4684 ));
4685 }
4686 }
4687 let agent_id = seat.agent.clone();
4688 self.state.seats.insert(seat.key.clone(), seat);
4689
4690 let mut post = None;
4691 let mut lines = None;
4692 let _coordination = self.coordinate(&wt).await?;
4693 if why.is_none() {
4694 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4695 if let Ok(r) = &rescued {
4696 self.state.note_withheld("graft", &r.withheld);
4697 }
4698 let head = git::rev_parse(&wt, "HEAD").await?;
4699 let dirty = match &rescued {
4703 Ok(r) => git::status_porcelain(&wt)
4704 .await
4705 .map(|st| {
4706 st.lines().any(|l| {
4707 !r.withheld
4708 .iter()
4709 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4710 })
4711 })
4712 .unwrap_or(true),
4713 Err(_) => true,
4714 };
4715 if dirty {
4716 why = Some("the fixer left changes that could not be committed".to_owned());
4717 } else if head == pre {
4718 why = Some("the fixer changed nothing".to_owned());
4719 } else {
4720 lines = git::changed_lines(&wt, &pre, &head).await;
4721 post = Some(head);
4722 }
4723 }
4724 let (adoption, adoption_error) = match &out {
4725 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4726 _ => (None, None),
4727 };
4728 let requested = graft::requested(&plan);
4729 let outcome = match why {
4730 None => {
4731 self.state.event(
4732 "graft",
4733 format!(
4734 "applied: {}",
4735 lines.map_or_else(
4736 || "changed lines unavailable".to_owned(),
4737 |n| format!("{n} changed lines")
4738 )
4739 ),
4740 );
4741 GraftOutcome::Applied
4742 }
4743 Some(why) => {
4744 crate::recovery::restore_repair(&wt, &pre)
4747 .await
4748 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4749 self.state.event(
4750 "graft",
4751 format!("rolled back: {why}; the winner is untouched"),
4752 );
4753 GraftOutcome::RolledBack(why)
4754 }
4755 };
4756 self.state.graft = Some(GraftRecord {
4757 pre,
4758 post,
4759 lines,
4760 limit: None,
4761 agent: Some(agent_id),
4762 outcome,
4763 requested,
4764 adoption,
4765 adoption_error,
4766 });
4767 self.state.save()?;
4768 Ok(())
4769 }
4770
4771 async fn fold_losers(&mut self) -> Result<()> {
4774 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4775 return Ok(());
4776 };
4777 if !self.state.candidate_preservation.is_empty() {
4778 bail!("candidate preservation incomplete; refusing cleanup");
4779 }
4780 let repo = self.state.repo.clone();
4781 let _coordination = self.coordinate(&repo).await?;
4782 let mut folded = Vec::new();
4783 for i in 0..self.state.candidates.len() {
4784 let c = &self.state.candidates[i];
4785 if c.label == winner || c.folded {
4786 continue;
4787 }
4788 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4789 remove_preserved_worktree(&repo, &wt).await?;
4790 if git::branch_exists(&repo, &branch).await?
4791 && !git::branch_delete(&repo, &branch).await?
4792 {
4793 bail!("candidate branch removal refused");
4794 }
4795 self.state.candidates[i].folded = true;
4796 self.state.save()?;
4797 folded.push(label.to_string());
4798 }
4799 let root = self.state.worktree_root();
4801 for j in 1..=self.roles.judges.len() {
4802 let wt = root.join(format!("judge-{j}"));
4803 if wt.exists() {
4804 remove_preserved_worktree(&repo, &wt).await?;
4805 }
4806 }
4807 if self.state.config.graph.advise {
4810 for k in 1..=self.state.config.graph.advisors {
4811 let wt = root.join(format!("advisor-{k}"));
4812 if wt.exists() {
4813 remove_preserved_worktree(&repo, &wt).await?;
4814 }
4815 }
4816 }
4817 if !folded.is_empty() {
4818 self.state
4819 .event("fold", format!("folded candidates {}", folded.join(", ")));
4820 self.state.save()?;
4821 }
4822 Ok(())
4823 }
4824
4825 async fn hold_base_sync(&mut self) -> Result<bool> {
4828 self.hold_base_sync_as(None).await
4829 }
4830
4831 async fn hold_base_sync_kind(
4832 &mut self,
4833 kind: crate::base_sync_recovery::StopKind,
4834 ) -> Result<()> {
4835 self.hold_base_sync_as(Some(kind)).await?;
4836 Ok(())
4837 }
4838
4839 async fn hold_base_sync_as(
4840 &mut self,
4841 kind: Option<crate::base_sync_recovery::StopKind>,
4842 ) -> Result<bool> {
4843 if !self.state.base_sync_blocked() {
4844 self.state.base_sync_recovery.stop = None;
4845 return Ok(false);
4846 }
4847 let head = match self.state.winner() {
4848 Some(w) => git::rev_parse(&w.worktree, "HEAD")
4849 .await
4850 .unwrap_or_default(),
4851 None => String::new(),
4852 };
4853 self.state.record_base_sync_stop(&head);
4854 if let Some(kind) = kind {
4855 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4856 }
4857 self.state.save()?;
4858 Ok(true)
4859 }
4860
4861 async fn sync_to_base(&mut self) -> Result<()> {
4903 if self.state.status == RunStatus::AlreadyInBase {
4904 return Ok(());
4905 }
4906 self.hold_base_sync().await?;
4907 let conflicted = self
4908 .state
4909 .base_sync
4910 .as_ref()
4911 .is_some_and(|s| s.conflict.is_some());
4912 let Some(winner) = self.state.winner().cloned() else {
4913 return Ok(());
4914 };
4915
4916 let repo = self.state.repo.clone();
4917 let remote = self.state.config.merge.remote.clone();
4918 let base_branch = self.state.base_branch.clone();
4919 let tracking = format!("{remote}/{base_branch}");
4920
4921 let fetched_base =
4922 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4923 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4928 return Ok(());
4929 };
4930
4931 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4932 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4933 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4934
4935 if (behind > 0 || conflicted || head == tip)
4944 && self
4945 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4946 .await?
4947 {
4948 return Ok(());
4949 }
4950 if conflicted && self.state.base_recovery_pending() {
4951 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4952 let owner =
4953 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4954 if !fetched_base
4955 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4956 || !git::is_clean(&winner.worktree).await?
4957 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4958 {
4959 return Ok(());
4960 }
4961 }
4962 let recovery = conflicted && self.state.consume_base_recovery(&head);
4963 if conflicted {
4964 self.state.save()?;
4966 if !recovery {
4967 return Ok(());
4968 }
4969 }
4970
4971 if behind == 0 {
4972 if let Some(from) = self
4981 .state
4982 .rebase_fixes
4983 .iter()
4984 .rev()
4985 .find_map(|r| r.from.clone())
4986 && from != head
4987 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4988 .await
4989 .is_ok_and(|o| o.ok())
4990 {
4991 git::sync_to_head(&winner.worktree).await?;
4992 }
4993 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4999 if let Some(why) = &conflict {
5000 self.state.status = RunStatus::Blocked;
5001 self.state.event("land", why.clone());
5002 }
5003 self.state.base_sync = Some(BaseSync {
5004 tip,
5005 behind: 0,
5006 attempts: attempts + usize::from(recovery),
5007 conflict,
5008 already_in: None,
5009 });
5010 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5011 .await?;
5012 self.state.save()?;
5013 return Ok(());
5014 }
5015
5016 if attempts >= BASE_SYNC_ROUNDS && !recovery {
5017 let why = format!(
5018 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5019 synchronization(s); synchronizing again would only race it",
5020 winner.branch
5021 );
5022 self.state.status = RunStatus::Blocked;
5023 self.state.base_sync = Some(BaseSync {
5024 tip,
5025 behind,
5026 attempts,
5027 conflict: Some(why.clone()),
5028 already_in: None,
5029 });
5030 self.state.event("land", why);
5031 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5032 .await?;
5033 self.state.save()?;
5034 return Ok(());
5035 }
5036
5037 if self.state.foreign_checkout.is_some() {
5038 let wait = repository_wait(&self.state, None);
5041 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5042 let merged = git::git_raw(
5043 &winner.worktree,
5044 &[
5045 "-c",
5046 "core.hooksPath=/dev/null",
5047 "merge",
5048 "--no-ff",
5049 "--no-autostash",
5050 "--no-edit",
5051 "-m",
5052 &format!("Merge {tracking} before verifying"),
5053 &tip,
5054 ],
5055 )
5056 .await?;
5057 let conflict = if merged.ok() {
5058 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5059 self.state.event(
5060 "land",
5061 format!("merged {tracking} into {} before verifying", winner.branch),
5062 );
5063 None
5064 } else {
5065 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5066 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5067 }
5068 let why = format!(
5069 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5070 winner.branch,
5071 merged.stderr.chars().take(600).collect::<String>()
5072 );
5073 self.state.status = RunStatus::Blocked;
5074 self.state.event("land", &why);
5075 Some(why)
5076 };
5077 let synced = conflict.is_none();
5078 self.state.base_sync = Some(BaseSync {
5079 tip,
5080 behind: if synced { 0 } else { behind },
5081 attempts: attempts + 1,
5082 conflict,
5083 already_in: None,
5084 });
5085 self.state.save()?;
5086 return Ok(());
5087 }
5088
5089 self.state.event(
5090 "land",
5091 format!(
5092 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5093 winner.branch
5094 ),
5095 );
5096 self.state.save()?;
5097
5098 let branch_tracking = format!("{remote}/{}", winner.branch);
5104 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5105 let remote_tip = if self.state.task_pr_branch().is_none()
5106 && matches!(&fetched_branch, Ok(o) if o.ok())
5107 {
5108 git::rev_parse(&repo, &branch_tracking).await.ok()
5109 } else {
5110 None
5111 };
5112 if let Some(theirs) = &remote_tip
5116 && !git::is_ancestor(&repo, theirs, &head).await
5117 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5118 .await
5119 .is_ok_and(|missing| missing.is_empty())
5120 {
5121 let why = format!(
5122 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5123 them",
5124 short(theirs),
5125 winner.branch
5126 );
5127 self.state.status = RunStatus::Blocked;
5128 self.state.base_sync = Some(BaseSync {
5129 tip,
5130 behind,
5131 attempts: attempts + usize::from(recovery),
5132 conflict: Some(why.clone()),
5133 already_in: None,
5134 });
5135 self.state.event("land", why);
5136 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5137 .await?;
5138 self.state.save()?;
5139 return Ok(());
5140 }
5141
5142 let scratch = self.state.dir().join("base-sync");
5143 let rebased =
5144 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5145 .await
5146 {
5147 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5148 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5149 Err(e) => Err(e),
5150 };
5151 let attempts = attempts + 1;
5152 let stop_kind = match &rebased {
5153 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5154 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5155 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5156 };
5157 match rebased {
5158 Ok(None) => {
5159 git::sync_to_head(&winner.worktree).await?;
5163 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5164 let mut conflict = None;
5165 if let Some(pinned) = &remote_tip {
5166 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5167 match pushed {
5168 Ok(o) if o.ok() => self.state.event(
5169 "land",
5170 format!("pushed rebased {} to {remote}", winner.branch),
5171 ),
5172 Ok(o) => {
5173 conflict = Some(format!(
5174 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5175 winner.branch,
5176 short(pinned),
5177 o.stderr.chars().take(600).collect::<String>()
5178 ));
5179 }
5180 Err(e) => {
5181 conflict = Some(format!(
5182 "rebased {} locally but could not push it: {e:#}",
5183 winner.branch
5184 ));
5185 }
5186 }
5187 }
5188 if let Some(why) = &conflict {
5189 self.state.status = RunStatus::Blocked;
5190 self.state.event("land", why.clone());
5191 }
5192 self.state.base_sync = Some(BaseSync {
5193 tip: tip.clone(),
5194 behind: 0,
5195 attempts,
5196 conflict,
5197 already_in: None,
5198 });
5199 self.state
5200 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5201 }
5202 Ok(Some(conflict)) => {
5203 let why = format!(
5204 "{} conflicts with {tracking} and did not rebase: {}",
5205 winner.branch,
5206 conflict.chars().take(600).collect::<String>()
5207 );
5208 self.state.status = RunStatus::Blocked;
5209 self.state.base_sync = Some(BaseSync {
5210 tip,
5211 behind,
5212 attempts,
5213 conflict: Some(why.clone()),
5214 already_in: None,
5215 });
5216 self.state.event("land", why);
5217 }
5218 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5222 self.state.save()?;
5223 return Ok(());
5224 }
5225 Err(e) => {
5226 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5227 self.state.status = RunStatus::Blocked;
5228 self.state.base_sync = Some(BaseSync {
5229 tip,
5230 behind,
5231 attempts,
5232 conflict: Some(why.clone()),
5233 already_in: None,
5234 });
5235 self.state.event("land", why);
5236 }
5237 }
5238 if !self.state.base_sync_blocked() {
5239 self.state.base_sync_recovery.stop = None;
5240 }
5241 self.hold_base_sync_kind(stop_kind).await?;
5242 self.state.save()?;
5243 Ok(())
5244 }
5245
5246 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5252 if self.state.task_pr_branch().is_some() {
5255 return None;
5256 }
5257 let repo = self.state.repo.clone();
5258 let remote = self.state.config.merge.remote.clone();
5259 let fetched = git::fetch(&repo, &remote, branch).await;
5260 if !matches!(&fetched, Ok(o) if o.ok()) {
5261 return None;
5262 }
5263 let branch_tracking = format!("{remote}/{branch}");
5264 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5265 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5266 return None;
5267 }
5268 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5269 .await
5270 .is_ok_and(|missing| missing.is_empty())
5271 {
5272 return Some(format!(
5273 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5274 them",
5275 short(&theirs)
5276 ));
5277 }
5278 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5279 Ok(o) if o.ok() => {
5280 self.state
5281 .event("land", format!("pushed rebased {branch} to {remote}"));
5282 None
5283 }
5284 Ok(o) => Some(format!(
5285 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5286 someone may have pushed): {}",
5287 short(&theirs),
5288 o.stderr.chars().take(600).collect::<String>()
5289 )),
5290 Err(e) => Some(format!(
5291 "{branch} is rebased locally but could not be pushed: {e:#}"
5292 )),
5293 }
5294 }
5295
5296 async fn settle_already_in(
5313 &mut self,
5314 branch: &str,
5315 tip: &str,
5316 head: &str,
5317 attempts: usize,
5318 behind: usize,
5319 ) -> Result<bool> {
5320 let repo = self.state.repo.clone();
5321 let remote = self.state.config.merge.remote.clone();
5322 let start = self.state.base_commit.clone();
5323 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5324 Ok(Some(e)) => e,
5325 Ok(None) => return Ok(false),
5326 Err(e) => {
5327 tracing::warn!("already-in-base check for {branch}: {e:#}");
5328 return Ok(false);
5329 }
5330 };
5331 let mut verified = vec![head.to_owned()];
5332 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5333 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5334 if matches!(&fetched, Ok(o) if o.ok())
5335 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5336 && theirs != head
5337 {
5338 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5339 Ok(Some(_)) => verified.push(theirs),
5340 _ => return Ok(false),
5341 }
5342 }
5343 let base_branch = self.state.base_branch.clone();
5344 let message = format!(
5345 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5346 land",
5347 evidence.names(),
5348 evidence.proof.as_str()
5349 );
5350 let closed =
5351 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5352 .await;
5353 match closed {
5354 Ok(Ok(url)) => self
5355 .state
5356 .event("land", format!("closed {url}: superseded on {base_branch}")),
5357 Ok(Err(why)) => self
5358 .state
5359 .event("land", format!("did not close a pull request: {why}")),
5360 Err(e) => {
5361 let why = format!(
5362 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5363 not be closed ({e:#}); resume to retry"
5364 );
5365 self.state.status = RunStatus::Blocked;
5366 self.state.base_sync = Some(BaseSync {
5367 tip: tip.to_owned(),
5368 behind,
5369 attempts,
5370 conflict: Some(why.clone()),
5371 already_in: None,
5372 });
5373 self.state.event("land", why);
5374 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5375 .await?;
5376 self.state.save()?;
5377 return Ok(true);
5378 }
5379 }
5380 self.state.status = RunStatus::AlreadyInBase;
5381 self.state.base_sync_recovery.stop = None;
5382 self.state.base_sync = Some(BaseSync {
5383 tip: tip.to_owned(),
5384 behind,
5385 attempts,
5386 conflict: None,
5387 already_in: Some(evidence),
5388 });
5389 self.state.event("land", message);
5390 self.state.save()?;
5391 self.settle_questions();
5392 Ok(true)
5393 }
5394
5395 fn landing_base(&self) -> String {
5405 self.state
5406 .base_sync
5407 .as_ref()
5408 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5409 }
5410
5411 pub async fn fix_selected(
5444 &mut self,
5445 ids: &[String],
5446 reason: &str,
5447 allow_stale: bool,
5448 ) -> Result<()> {
5449 let reason = reason.trim();
5450 if reason.is_empty() {
5451 bail!("a fix request needs a reason — that is the operator's own record of why");
5452 }
5453 if ids.is_empty() {
5454 bail!("no finding id given");
5455 }
5456 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5457 bail!(
5458 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5459 has already concluded — can be given a targeted fix. A run still \
5460 in progress should simply be resumed; a `merged` run's branch has \
5461 already landed, so its answer is a fresh `magi review <branch>`, \
5462 not reopening this run's own record",
5463 self.state.id,
5464 self.state.status.as_str()
5465 );
5466 }
5467 let Some(winner) = self.state.winner().cloned() else {
5468 bail!("run {} has no winning candidate to fix", self.state.id);
5469 };
5470 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5471 bail!(
5472 "branch `{}` no longer exists; this run cannot be extended",
5473 winner.branch
5474 );
5475 }
5476 let home = crate::run::home();
5477 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5478 bail!(
5479 "run {} is currently being worked on by another magi process",
5480 self.state.id
5481 );
5482 }
5483 let _claim = FixClaim::acquire(&self.state.dir())?;
5489
5490 let mut seen = BTreeSet::new();
5494 let mut findings = Vec::new();
5495 let mut missing = Vec::new();
5496 for id in ids {
5497 if !seen.insert(id.clone()) {
5498 continue;
5499 }
5500 match self.state.finding(id) {
5501 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5502 id: f.id.clone(),
5503 severity: f.severity,
5504 reviewer_vote: rec.vote,
5505 round: round.round,
5506 round_head: round.head.clone(),
5507 reviewer: rec.reviewer,
5508 agent: rec.agent.clone(),
5509 file: f.file.clone(),
5510 line: f.line,
5511 title: f.title.clone(),
5512 detail: f.detail.clone(),
5513 outcome: OperatorFixOutcome::Pending,
5514 }),
5515 None => missing.push(id.clone()),
5516 }
5517 }
5518 if !missing.is_empty() {
5519 bail!(
5520 "unknown finding id(s): {}; nothing was changed",
5521 missing.join(", ")
5522 );
5523 }
5524
5525 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5526 let stale_details: Vec<(String, String)> = findings
5527 .iter()
5528 .filter(|f| f.round_head != head_at_request)
5529 .map(|f| (f.id.clone(), f.round_head.clone()))
5530 .collect();
5531 let stale = !stale_details.is_empty();
5532 if stale && !allow_stale {
5533 bail!(
5534 "the branch has moved since some finding(s) were raised — {} — now \
5535 at {}; pass --allow-stale to fix anyway, or re-run review first",
5536 stale_details
5537 .iter()
5538 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5539 .collect::<Vec<_>>()
5540 .join(", "),
5541 short(&head_at_request)
5542 );
5543 }
5544
5545 let request = OperatorFixRequest {
5546 requested_at: Timestamp::now(),
5547 reason: reason.to_owned(),
5548 findings,
5549 head_at_request: head_at_request.clone(),
5550 allow_stale,
5551 stale,
5552 fix: None,
5553 result_head: None,
5554 follow_up_review_run: None,
5555 };
5556 self.state.event(
5557 "fix",
5558 format!(
5559 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5560 request.findings.len(),
5561 request
5562 .findings
5563 .iter()
5564 .map(|f| f.id.as_str())
5565 .collect::<Vec<_>>()
5566 .join(", "),
5567 ),
5568 );
5569 self.state.operator_fixes.push(request);
5576 self.state.save()?;
5577 let request_index = self.state.operator_fixes.len() - 1;
5578
5579 let repo = self.state.repo.clone();
5588 let coordination = self.coordinate(&repo).await?;
5589 if winner.worktree.exists() {
5590 let dirty = git::git(
5593 &winner.worktree,
5594 &["status", "--porcelain", "--untracked-files=all"],
5595 )
5596 .await?;
5597 let only_withheld = dirty.lines().all(|l| {
5598 l.strip_prefix("?? ")
5599 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5600 });
5601 if !only_withheld {
5602 bail!(
5603 "`{}` has uncommitted changes; refusing to touch it — commit or \
5604 discard them first",
5605 winner.worktree.display()
5606 );
5607 }
5608 git::worktree_remove(&self.state.repo, &winner.worktree)
5609 .await
5610 .ok();
5611 }
5612 let fix_worktree = self.state.worktree_root().join("operator-fix");
5613 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5614 git::git(
5615 &self.state.repo,
5616 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5617 )
5618 .await
5619 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5620 if !git::is_clean(&fix_worktree).await? {
5621 git::worktree_remove(&self.state.repo, &fix_worktree)
5622 .await
5623 .ok();
5624 bail!(
5625 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5626 winner.branch
5627 );
5628 }
5629 let repo_for_setup = self.state.repo.clone();
5630 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5631 Ok(()) => {
5632 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5633 }
5634 Err(e) => Err(e),
5635 };
5636 if let Err(e) = setup {
5637 git::worktree_remove(&self.state.repo, &fix_worktree)
5638 .await
5639 .ok();
5640 return Err(e);
5641 }
5642
5643 self.state.resource_block = None;
5646
5647 let run_id = self.state.id.clone();
5648 let prompts = self.state.config.prompts.clone();
5649 let language = self.state.config.graph.language.clone();
5650 let sessions = self.state.config.graph.sessions;
5651 let artifacts = agent::artifacts_dir(&self.state.dir());
5652 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5653 .findings
5654 .iter()
5655 .map(|f| Finding {
5656 id: f.id.clone(),
5657 severity: f.severity,
5658 file: f.file.clone(),
5659 line: f.line,
5660 title: f.title.clone(),
5661 detail: f.detail.clone(),
5662 })
5663 .collect();
5664 let fix_prompt = prompt::operator_fix_in(
5665 &self.state.instruction,
5666 &finding_list,
5667 reason,
5668 &stale_details,
5669 &head_at_request,
5670 &language,
5671 &self.state.config.graph.github_language,
5672 );
5673 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5674 drop(coordination);
5675 let (job, seat, out) = self
5676 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5677 prompt: fix_prompt.clone(),
5678 spec,
5679 seat,
5680 cwd: fix_worktree.clone(),
5681 timeout,
5682 allow_write: true,
5683 sessions,
5684 artifacts: artifacts.clone(),
5685 stem: "operator-fix".to_owned(),
5686 handover: None,
5687 })
5688 .await;
5689 let agent_id = seat.agent.clone();
5690
5691 let mut fix = FixRecord {
5692 agent: agent_id,
5693 addressed: Vec::new(),
5694 rejected: Vec::new(),
5695 notes: String::new(),
5696 committed: false,
5697 failed: None,
5698 duration_ms: 0,
5699 continuation: None,
5700 };
5701 let mut final_seat = seat.clone();
5702 match out {
5703 AgentOutcome::Ok(o) => {
5704 fix.duration_ms = o.duration_ms;
5705 let parsed = verdict::extract_json::<FixReport>(&o.text);
5706 let incomplete_reason = match &parsed {
5707 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5708 "the reply parsed, but it reported a command whose own CLI \
5709 never confirmed an exit status"
5710 .to_owned(),
5711 ),
5712 Ok(_) => None,
5713 Err(e) => Some(e.to_string()),
5714 };
5715 match incomplete_reason {
5716 None => {
5717 let report = parsed.expect("checked Ok above");
5718 fix.addressed = report.addressed;
5719 fix.rejected = report.rejected;
5720 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5721 }
5722 Some(reason) => {
5723 let (resumed_seat, resolved, failure, cont) = self
5724 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5725 .await;
5726 fix.duration_ms += cont.cumulative_wait_ms;
5727 fix.continuation = Some(cont);
5728 final_seat = resumed_seat;
5729 match resolved {
5730 Some(report) => {
5731 fix.addressed = report.addressed;
5732 fix.rejected = report.rejected;
5733 fix.notes =
5734 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5735 }
5736 None => fix.failed = failure,
5737 }
5738 }
5739 }
5740 }
5741 AgentOutcome::Dropped(o) => {
5742 fix.duration_ms = o.duration_ms;
5743 let why = o
5744 .dropped
5745 .as_ref()
5746 .map(|d| d.why.as_str())
5747 .unwrap_or("the CLI ended the stream without delivering its answer");
5748 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5749 }
5750 AgentOutcome::Quota(o) => {
5751 self.state.quota.push(QuotaLoss::from_report(
5752 final_seat.key.clone(),
5753 "fix",
5754 o.quota.as_ref(),
5755 ));
5756 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5757 }
5758 AgentOutcome::Failed(e) => fix.failed = Some(e),
5759 AgentOutcome::ResourceBlocked(e) => {
5760 fix.failed = Some(format!("the fixer was not started: {e}"));
5761 }
5762 }
5763 if fix.continuation.is_none() {
5766 fix.continuation = Some(ContinuationRecord::not_needed());
5767 }
5768 self.state.seats.insert(final_seat.key.clone(), final_seat);
5769
5770 let rescue_message = format!(
5771 "magi: operator-selected fix ({}) (uncommitted work)",
5772 self.state.operator_fixes[request_index]
5773 .findings
5774 .iter()
5775 .map(|f| f.id.as_str())
5776 .collect::<Vec<_>>()
5777 .join(", ")
5778 );
5779 let repo = self.state.repo.clone();
5780 let coordination = self.coordinate(&repo).await?;
5781 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5782 self.state.note_withheld("fix", &r.withheld);
5783 }
5784 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5785 fix.committed = after != head_at_request;
5786 git::worktree_remove(&self.state.repo, &fix_worktree)
5787 .await
5788 .ok();
5789 drop(coordination);
5790
5791 self.state.event(
5792 "fix",
5793 match &fix.failed {
5794 Some(reason) => format!(
5795 "operator fix: adoption report was lost ({reason}); {}",
5796 if fix.committed {
5797 "committed"
5798 } else {
5799 "NO new commit"
5800 }
5801 ),
5802 None => format!(
5803 "operator fix: {} addressed, {} rejected, {}",
5804 fix.addressed.len(),
5805 fix.rejected.len(),
5806 if fix.committed {
5807 "committed"
5808 } else {
5809 "NO new commit"
5810 }
5811 ),
5812 },
5813 );
5814
5815 for f in &mut self.state.operator_fixes[request_index].findings {
5822 f.outcome = if fix.failed.is_some() {
5823 OperatorFixOutcome::Unreported
5824 } else if fix.addressed.contains(&f.id) {
5825 OperatorFixOutcome::Addressed
5826 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5827 OperatorFixOutcome::Rejected { why: r.why.clone() }
5828 } else {
5829 OperatorFixOutcome::Unreported
5830 };
5831 }
5832
5833 let committed = fix.committed;
5834 if committed {
5835 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5836 }
5837 self.state.operator_fixes[request_index].fix = Some(fix);
5838 self.state.save()?;
5841 if self.state.resource_block.is_some() {
5842 return self.stop_for_resource();
5843 }
5844
5845 if committed {
5846 self.state.event(
5847 "fix",
5848 format!(
5849 "operator fix committed {}; opening a follow-up review-only run",
5850 short(&after)
5851 ),
5852 );
5853 let origin =
5856 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5857 match Self::review(
5858 &self.state.repo,
5859 &winner.branch,
5860 self.state.config.clone(),
5861 origin,
5862 )
5863 .await
5864 {
5865 Ok(mut follow_up) => {
5866 follow_up.state.event(
5867 "start",
5868 format!(
5869 "requested by an operator fix on run {} for finding(s) {}",
5870 self.state.id,
5871 self.state.operator_fixes[request_index]
5872 .findings
5873 .iter()
5874 .map(|f| f.id.as_str())
5875 .collect::<Vec<_>>()
5876 .join(", "),
5877 ),
5878 );
5879 follow_up.state.save()?;
5880 let follow_up_id = follow_up.state.id.clone();
5881 let adopted = match crate::direct::adopt(
5884 &follow_up.state,
5885 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5886 ) {
5887 Ok(a) => a,
5888 Err(e) => {
5889 self.state.event(
5892 "fix",
5893 format!(
5894 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5895 ),
5896 );
5897 self.state.save()?;
5898 return Ok(());
5899 }
5900 };
5901 let executed = follow_up.execute().await;
5902 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5903 if let Some(a) = adopted {
5904 a.finish(&follow_up.state, executed);
5905 }
5906 if let Some(e) = failure {
5907 self.state.event(
5908 "fix",
5909 format!(
5910 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5911 ),
5912 );
5913 }
5914 self.state.operator_fixes[request_index].follow_up_review_run =
5915 Some(follow_up_id);
5916 }
5917 Err(e) => {
5918 self.state.event(
5919 "fix",
5920 format!("committed the fix but could not open a follow-up review: {e:#}"),
5921 );
5922 }
5923 }
5924 self.state.save()?;
5925 }
5926
5927 Ok(())
5928 }
5929
5930 async fn ask_fixer(
5950 &mut self,
5951 winner: &Candidate,
5952 node: &'static str,
5953 round: Option<usize>,
5954 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5955 ) -> (SeatJob, SeatState, AgentOutcome) {
5956 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5957 self.ask_fixer_among(attempts, node, round, None, build)
5958 .await
5959 }
5960
5961 async fn ask_fixer_among(
5966 &mut self,
5967 attempts: Vec<(AgentSpec, String)>,
5968 node: &'static str,
5969 round: Option<usize>,
5970 restore: Option<(&Path, &str)>,
5971 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5972 ) -> (SeatJob, SeatState, AgentOutcome) {
5973 let run_id = self.state.id.clone();
5974 let prompts = self.state.config.prompts.clone();
5975 let cache = self.state.config.cache_dir();
5976 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5977 let mut last = None;
5978 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5979 let seat = self.seat(&key, &spec.id);
5980 let mut job = build(spec.clone(), seat);
5981 if i > 0 {
5982 job.stem = format!("{}-{}", job.stem, spec.id);
5983 }
5984 let ctx = WaveCtx {
5985 carry_seats: false,
5986 pauses: Some((&self.pause, &self.interrupt)),
5987 run: &run_id,
5988 node,
5989 prompts: &prompts,
5990 cache: cache.as_deref(),
5991 round,
5992 };
5993 let (seat, out) =
5994 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5995 let advances = match &out {
5998 AgentOutcome::Ok(o) => agent::output_advances(o),
5999 AgentOutcome::ResourceBlocked(_) => false,
6000 _ => true,
6001 };
6002 if let Some(next) = ids.get(i + 1)
6003 && advances
6004 {
6005 self.state.seats.insert(seat.key.clone(), seat.clone());
6006 let class =
6007 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6008 let reason = match &out {
6009 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6010 other => fail_reason(other),
6011 };
6012 if let Some((wt, rev)) = restore
6016 && let Err(e) = self.graft_restore(wt, rev).await
6017 {
6018 let why =
6019 format!("could not restore the worktree before the next fixer: {e:#}");
6020 tracing::warn!("{why}");
6021 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6022 AgentOutcome::ResourceBlocked(why)
6023 } else {
6024 AgentOutcome::Failed(why)
6025 };
6026 last = Some((job, seat, out));
6027 break;
6028 }
6029 record_handover(
6030 &mut self.state,
6031 node,
6032 &seat.key,
6033 &spec.id,
6034 next,
6035 &class,
6036 &reason,
6037 );
6038 continue;
6039 }
6040 last = Some((job, seat, out));
6041 break;
6042 }
6043 let (job, seat, out) = last.expect("the fixer chain always has an entry");
6044 if let AgentOutcome::Quota(output) = &out
6045 && node != "graft"
6046 {
6047 let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6052 block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6053 self.state.quota.push(loss);
6054 self.state.seats.insert(seat.key.clone(), seat.clone());
6055 self.state.status = RunStatus::Blocked;
6056 return (
6057 job,
6058 seat,
6059 AgentOutcome::ResourceBlocked(
6060 "fixer unavailable; restore agent availability and resume".into(),
6061 ),
6062 );
6063 }
6064 (job, seat, out)
6065 }
6066
6067 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6078 use crate::recovery::{Mode, Repair};
6079 if !self.state.reviews.is_empty() {
6080 return Ok(());
6081 }
6082 let Some(mut rec) = self.state.recovery.clone() else {
6083 return Ok(());
6084 };
6085 if rec.mode != Mode::VerifyRepair {
6086 return Ok(());
6087 }
6088 let wt = winner.worktree.clone();
6089 let mut head = git::rev_parse(&wt, "HEAD").await?;
6090 match rec.repair.clone() {
6091 Repair::Done { .. } => return Ok(()),
6092 Repair::WaitingForRestore { before } => {
6093 self.graft_restore(&wt, &before).await?;
6094 head = before;
6095 rec.repair = Repair::NotStarted;
6096 self.state.recovery = Some(rec.clone());
6097 self.state.save()?;
6098 }
6099 Repair::Started { before } => {
6100 if let Err(e) = self.graft_restore(&wt, &before).await {
6104 if e.is::<crate::repo_lock::ResourceBlocked>() {
6105 return Err(e);
6106 }
6107 return self
6108 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6109 .await;
6110 }
6111 rec.repair = Repair::Done {
6112 before: before.clone(),
6113 after: before,
6114 committed: false,
6115 failed: Some("interrupted; rolled back".to_owned()),
6116 };
6117 self.state
6118 .event("recovery", "the repair was interrupted and rolled back");
6119 self.state.recovery = Some(rec);
6120 self.state.save()?;
6121 return Ok(());
6122 }
6123 Repair::NotStarted => {}
6124 }
6125 if crate::recovery::recheck(&mut rec, &head) {
6126 self.state.event("recovery", rec.summary());
6127 self.state.recovery = Some(rec);
6128 self.state.save()?;
6129 return Ok(());
6130 }
6131 rec.repair = Repair::Started {
6132 before: head.clone(),
6133 };
6134 self.state.recovery = Some(rec.clone());
6135 self.state.save()?;
6136
6137 let language = self.state.config.graph.language.clone();
6138 let sessions = self.state.config.graph.sessions;
6139 let artifacts = agent::artifacts_dir(&self.state.dir());
6140 let fix_prompt = prompt::recovery_fix(
6141 &self.state.instruction,
6142 &rec,
6143 self.state.config.graph.review_rounds,
6144 &language,
6145 );
6146 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6147 self.state.event(
6148 "recovery",
6149 format!(
6150 "repair: the fixer starts from the failed verification on {}",
6151 short(&head)
6152 ),
6153 );
6154 let (_job, seat, out) = self
6155 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6156 prompt: fix_prompt.clone(),
6157 spec,
6158 seat,
6159 cwd: wt.clone(),
6160 timeout,
6161 allow_write: true,
6162 sessions,
6163 artifacts: artifacts.clone(),
6164 stem: "fix-recovery".to_owned(),
6165 handover: None,
6166 })
6167 .await;
6168 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6169 rec.repair = Repair::WaitingForRestore { before: head };
6170 self.state.recovery = Some(rec);
6171 return self.stop_for_resource();
6172 }
6173 let failed = match &out {
6174 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6175 AgentOutcome::Quota(o) => {
6176 self.state.quota.push(QuotaLoss::from_report(
6177 seat.key.clone(),
6178 "fix",
6179 o.quota.as_ref(),
6180 ));
6181 Some("rate limited (quota); the fixer could not run".to_owned())
6182 }
6183 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6184 AgentOutcome::Failed(e) => Some(e.clone()),
6185 };
6186 self.state.seats.insert(seat.key.clone(), seat);
6187 let coordination = self.coordinate(&wt).await?;
6188 let failed = match failed {
6189 Some(why) => {
6191 if let Err(e) = git::restore_to(&wt, &head).await {
6192 return self
6193 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6194 .await;
6195 }
6196 Some(why)
6197 }
6198 None => {
6199 if let Ok(r) = git::rescue_commit(
6200 &wt,
6201 "magi: recovery repair of the failed verification (uncommitted work)",
6202 )
6203 .await
6204 {
6205 self.state.note_withheld("fix", &r.withheld);
6206 }
6207 None
6208 }
6209 };
6210 let after = git::rev_parse(&wt, "HEAD").await?;
6211 drop(coordination);
6212 self.state.event(
6213 "recovery",
6214 match &failed {
6215 Some(why) => format!("repair: nothing kept ({why})"),
6216 None if after != head => {
6217 format!("repair: committed {}", short(&after))
6218 }
6219 None => "repair: the fixer made no commit".to_owned(),
6220 },
6221 );
6222 rec.repair = Repair::Done {
6223 before: head.clone(),
6224 committed: after != head,
6225 after,
6226 failed,
6227 };
6228 self.state.recovery = Some(rec);
6229 self.state.save()?;
6230 Ok(())
6231 }
6232
6233 async fn stop_recovery(
6236 &mut self,
6237 mut rec: crate::recovery::RecoveryRecord,
6238 head: &str,
6239 why: String,
6240 ) -> Result<()> {
6241 rec.repair = crate::recovery::Repair::Started {
6243 before: head.to_owned(),
6244 };
6245 self.state.recovery = Some(rec);
6246 self.state
6247 .event("recovery", format!("repair could not be undone: {why}"));
6248 self.state.status = RunStatus::Blocked;
6249 self.state.save()?;
6250 bail!("recovery repair could not be rolled back: {why}")
6251 }
6252
6253 async fn review_loop(&mut self) -> Result<()> {
6254 if self.hold_base_sync().await? {
6255 return Ok(());
6256 }
6257 loop {
6258 if self
6259 .state
6260 .base_sync
6261 .as_ref()
6262 .is_some_and(|s| s.conflict.is_some())
6263 {
6264 return Ok(());
6265 }
6266 if self.catchup_repairable() {
6267 let record = self.state.reviews.last().expect("repairable round").clone();
6268 let winner = self
6269 .state
6270 .winner()
6271 .cloned()
6272 .context("catch-up repair has no winner")?;
6273 if !self.fix_review_round(&winner, record).await? {
6274 return Ok(());
6275 }
6276 }
6277 self.review_loop_pass().await?;
6278 if !self.catchup_repairable() {
6279 return Ok(());
6280 }
6281 }
6282 }
6283
6284 fn catchup_repairable(&self) -> bool {
6287 self.state.reviews.len() < self.state.config.graph.review_rounds
6288 && self.state.reviews.last().is_some_and(|r| {
6289 !r.clean
6290 && r.blocking == 0
6291 && r.fix.is_none()
6292 && r.e2e_status() == E2eStatus::Failed
6293 })
6294 }
6295
6296 async fn review_loop_pass(&mut self) -> Result<()> {
6297 if self
6302 .state
6303 .base_sync
6304 .as_ref()
6305 .is_some_and(|s| s.conflict.is_some())
6306 {
6307 return Ok(());
6308 }
6309 let run_id = self.state.id.clone();
6314 let prompts = self.state.config.prompts.clone();
6315 let Some(winner) = self.state.winner().cloned() else {
6316 return Ok(());
6317 };
6318 let max_rounds = self.state.config.graph.review_rounds;
6319 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6329 if status == RunStatus::Gating {
6332 self.record_contested_handoff();
6333 }
6334 self.state.status = status;
6335 self.state.save()?;
6336 return Ok(());
6337 }
6338 self.state.status = RunStatus::Reviewing;
6339 if self.state.reviews.last().is_some_and(|r| {
6349 r.e2e_status() == E2eStatus::ResourceBlocked
6350 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6351 }) {
6352 let shell = self.state.config.shell();
6353 return self
6354 .stop_unresolved_review(
6355 "the last round's own verification never resolved",
6356 &shell,
6357 &winner.worktree,
6358 )
6359 .await;
6360 }
6361
6362 self.recovery_repair(&winner).await?;
6363 if self.resource_halt()? {
6364 return Ok(());
6365 }
6366
6367 let repo = self.state.repo.clone();
6368 let root = self.state.worktree_root();
6369 let language = self.state.config.graph.language.clone();
6370 let sessions = self.state.config.graph.sessions;
6371 let artifacts = self.merge_repair_artifacts();
6372 let base = self.landing_base();
6373 let base_short = short(&base);
6374 let reviewers = self.roles.reviewers.clone();
6375 let shell = self.state.config.shell();
6376
6377 for round in (self.state.reviews.len() + 1)..=max_rounds {
6378 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6379 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6380 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6381 let prev_verification = self
6390 .state
6391 .reviews
6392 .last()
6393 .and_then(|r| r.verification_summary(&head));
6394 let prior_section = match (&self.state.recovery, round) {
6397 (Some(rec), 1) if self.state.reviews.is_empty() => {
6398 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6399 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6400 Some(crate::recovery::head_of(
6401 &git::diff(&winner.worktree, reviewed, "HEAD")
6402 .await
6403 .unwrap_or_default(),
6404 crate::recovery::SINCE_DIFF_MAX,
6405 ))
6406 }
6407 _ => None,
6408 };
6409 Some(prompt::recovery_review(rec, since.as_deref()))
6410 }
6411 _ => None,
6412 };
6413
6414 let mut jobs = Vec::new();
6418 let coordination = self.coordinate(&repo).await?;
6419 for (r, spec) in reviewers.iter().cloned().enumerate() {
6420 let wt = root.join(format!("review-{}", r + 1));
6421 if wt.exists() {
6424 git::reset_detached(&wt, &head).await?;
6425 } else {
6426 git::worktree_add_detached(&repo, &wt, &head).await?;
6427 }
6428 self.setup_seat_worktree(&repo, &wt, false).await?;
6429 let seat_key = format!("review-{}", r + 1);
6430 let spec = pick_start_spec(
6433 &self.roles.reviewer_roster,
6434 spec,
6435 self.state.seat_history.get(&seat_key),
6436 );
6437 let seat = self.seat(&seat_key, &spec.id);
6438 jobs.push(SeatJob {
6439 prompt: prompt::review(&prompt::ReviewCtx {
6440 instruction: &self.state.instruction,
6441 branch: &winner.branch,
6442 base_short: &base_short,
6443 stat: &stat,
6444 patch: &patch,
6445 verification: prev_verification.as_ref(),
6446 prior: prior_section.as_deref(),
6447 reviewers: reviewers.len(),
6448 round,
6449 rounds: max_rounds,
6450 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6453 lens: Lens::for_seat(r),
6454 language: &language,
6455 github_language: &self.state.config.graph.github_language,
6456 }),
6457 spec,
6458 seat,
6459 cwd: wt,
6460 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6461 allow_write: false,
6462 sessions,
6463 artifacts: artifacts.clone(),
6464 stem: format!("review-{round}-{}", r + 1),
6465 handover: None,
6466 });
6467 }
6468
6469 drop(coordination);
6470 self.state.event(
6471 "review",
6472 format!(
6473 "round {round}: {} reviewers on {}",
6474 jobs.len(),
6475 short(&head)
6476 ),
6477 );
6478 let mut quota_losses = Vec::new();
6479 let review_retries = self.state.config.graph.retries;
6480 let review_cache = self.state.config.cache_dir();
6481 let ctx = WaveCtx {
6482 carry_seats: true,
6483 pauses: Some((&self.pause, &self.interrupt)),
6484 run: &run_id,
6485 node: "review",
6486 prompts: &prompts,
6487 cache: review_cache.as_deref(),
6488 round: Some(round),
6489 };
6490 let results = ask_json_wave::<Review>(
6491 jobs,
6492 Arc::clone(&self.sem),
6493 review_retries,
6494 &self.roles.reviewer_roster,
6495 &ctx,
6496 &mut quota_losses,
6497 &mut self.state,
6498 &|_: &Review| Ok(()),
6499 )
6500 .await;
6501 if self.state.resource_block.is_some() {
6502 return self.stop_for_resource();
6503 }
6504 let round_quota_missing = quota_losses.len();
6508 self.state.quota.extend(quota_losses);
6509
6510 let mut records = Vec::new();
6511 let mut all_findings = Vec::new();
6512 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6513 let agent_id = seat.agent.clone();
6514 self.state.seats.insert(seat.key.clone(), seat);
6515 let mut record = ReviewRecord {
6516 reviewer: r + 1,
6517 agent: agent_id,
6518 summary: String::new(),
6519 findings: Vec::new(),
6520 vote: None,
6521 failed: None,
6522 duration_ms: 0,
6523 attempts,
6529 };
6530 match res {
6531 Ok((review, out)) => {
6532 record.summary =
6540 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6541 record.vote = Some(review.vote);
6542 record.duration_ms = out.duration_ms;
6543 for (n, mut f) in review.findings.into_iter().enumerate() {
6544 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6547 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6548 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6549 f.file = f
6555 .file
6556 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6557 all_findings.push(f.clone());
6558 record.findings.push(f);
6559 }
6560 self.state.event(
6561 "review",
6562 format!(
6563 "round {round}: reviewer {} voted {} with {} finding(s)",
6564 r + 1,
6565 review.vote.label(),
6566 record.findings.len()
6567 ),
6568 );
6569 }
6570 Err(e) => {
6571 record.failed = Some(e.to_string());
6572 self.state.event(
6573 "review",
6574 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6575 );
6576 }
6577 }
6578 records.push(record);
6579 }
6580
6581 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6588 let vote_split =
6589 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6590 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6591 if vote_split {
6592 self.state.event(
6593 "review",
6594 format!(
6595 "round {round}: votes split ({}) — one round of reconsideration",
6596 initial_votes
6597 .iter()
6598 .map(|v| v.label())
6599 .collect::<Vec<_>>()
6600 .join(", ")
6601 ),
6602 );
6603 let panel: Vec<ReviewSeatReport<'_>> = records
6606 .iter()
6607 .filter_map(|r| {
6608 r.vote.map(|vote| ReviewSeatReport {
6609 reviewer: r.reviewer,
6610 vote,
6611 summary: &r.summary,
6612 findings: &r.findings,
6613 })
6614 })
6615 .collect();
6616
6617 let mut jobs = Vec::new();
6618 let mut seats_at = Vec::new();
6619 for (r, spec) in reviewers.iter().cloned().enumerate() {
6620 if records[r].vote.is_none() {
6624 continue;
6625 }
6626 let wt = root.join(format!("review-{}", r + 1));
6627 let seat_key = format!("review-{}", r + 1);
6628 let spec = self.occupant(&seat_key, spec);
6629 let seat = self.seat(&seat_key, &spec.id);
6630 let build = |with_patch: bool| {
6638 prompt::review_reconsider(&ReviewReconsiderCtx {
6639 instruction: &self.state.instruction,
6640 reviewer: r + 1,
6641 lens: Lens::for_seat(r),
6642 panel: &panel,
6643 patch: with_patch.then_some(ReviewPatch {
6644 branch: &winner.branch,
6645 base_short: &base_short,
6646 stat: &stat,
6647 patch: &patch,
6648 }),
6649 round,
6650 rounds: max_rounds,
6651 language: &language,
6652 })
6653 };
6654 let full = build(true);
6655 let prompt = if has_context(&spec, &seat, sessions) {
6656 build(false)
6657 } else {
6658 full.clone()
6659 };
6660 jobs.push(SeatJob {
6661 prompt,
6662 spec,
6663 seat,
6664 cwd: wt,
6665 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6666 allow_write: false,
6667 sessions,
6668 artifacts: artifacts.clone(),
6669 stem: format!("review-{round}-reconsider-{}", r + 1),
6670 handover: Some(full),
6671 });
6672 seats_at.push(r);
6673 }
6674
6675 let mut recon_quota_losses = Vec::new();
6676 let recon_cache = self.state.config.cache_dir();
6677 let recon_ctx = WaveCtx {
6678 carry_seats: true,
6679 pauses: Some((&self.pause, &self.interrupt)),
6680 run: &run_id,
6681 node: "review",
6682 prompts: &prompts,
6683 cache: recon_cache.as_deref(),
6684 round: Some(round),
6685 };
6686 let recon_results = ask_json_wave::<ReviewRevote>(
6687 jobs,
6688 Arc::clone(&self.sem),
6689 review_retries,
6690 &self.roles.reviewer_roster,
6691 &recon_ctx,
6692 &mut recon_quota_losses,
6693 &mut self.state,
6694 &|_: &ReviewRevote| Ok(()),
6695 )
6696 .await;
6697 if self.state.resource_block.is_some() {
6698 return self.stop_for_resource();
6699 }
6700 self.state.quota.extend(recon_quota_losses);
6701
6702 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6703 let agent_id = seat.agent.clone();
6704 self.state.seats.insert(seat.key.clone(), seat);
6705 let mut rec = ReviewRevoteRecord {
6706 reviewer: r + 1,
6707 agent: agent_id,
6708 vote: None,
6709 reason: String::new(),
6710 failed: None,
6711 };
6712 match res {
6713 Ok((rv, _)) => {
6714 rec.vote = Some(rv.vote);
6715 rec.reason =
6716 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6717 self.state.event(
6718 "review",
6719 format!(
6720 "round {round}: reviewer {} revoted {}",
6721 r + 1,
6722 rv.vote.label()
6723 ),
6724 );
6725 }
6726 Err(e) => {
6727 rec.failed = Some(e.to_string());
6728 self.state.event(
6729 "review",
6730 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6731 );
6732 }
6733 }
6734 reconsideration.push(rec);
6735 }
6736 } else if initial_votes.len() > 1 {
6737 self.state.event(
6738 "review",
6739 format!(
6740 "round {round}: votes agreed ({}) — no reconsideration",
6741 initial_votes[0].label()
6742 ),
6743 );
6744 }
6745
6746 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6747 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6748 let defer_e2e =
6759 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6760 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6761 let reason =
6762 format!("{blocking} blocking finding(s) already required a fix this round");
6763 self.state.event(
6764 "verify",
6765 format!(
6766 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6767 {}); it will run once a round has none left",
6768 short(&head)
6769 ),
6770 );
6771 (Vec::new(), false, true, Some(reason))
6772 } else {
6773 let e2e_commands = self.state.config.verify.e2e.clone();
6774 let cache_dir = self.state.config.cache_dir();
6775 let context = format!("round {round}");
6776 let wait = self.repository_wait();
6777 let (e2e, verify_retried) = with_cache_lease(
6778 &mut self.state,
6779 cache_dir.as_deref(),
6780 "e2e",
6781 "e2e",
6782 &winner.worktree,
6783 &head,
6784 verify_timeout,
6785 &context,
6786 &wait,
6787 |state, budget| {
6788 let shell = shell.clone();
6789 let e2e_commands = e2e_commands.clone();
6790 let worktree = winner.worktree.clone();
6791 let context = context.clone();
6792 async move {
6793 run_e2e_with_retry(
6794 state,
6795 &shell,
6796 &e2e_commands,
6797 &worktree,
6798 budget,
6799 &context,
6800 )
6801 .await
6802 }
6803 },
6804 )
6805 .await?;
6806 (e2e, verify_retried, false, None)
6807 };
6808
6809 let expected = records.len();
6810 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6811 let incomplete = answered < expected;
6812 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6813 let policy = self.state.config.graph.incomplete_review;
6814 let clean = round_is_clean(
6815 blocking,
6816 e2e_ok,
6817 answered,
6818 expected,
6819 round_quota_missing,
6820 policy,
6821 );
6822
6823 let mut round_record = ReviewRound {
6824 round,
6825 head: head.clone(),
6826 verified_head: None,
6827 verified_at: None,
6828 reviews: records,
6829 e2e,
6830 verify_retried,
6831 e2e_deferred,
6832 e2e_defer_reason,
6833 fix: None,
6834 blocking,
6835 answered,
6836 expected,
6837 clean,
6838 progressed: false,
6839 vote_split,
6840 reconsideration,
6841 verdict: None,
6842 };
6843 round_record.verdict = ReviewVote::worst(
6847 round_record
6848 .final_votes()
6849 .into_iter()
6850 .map(|(_, _, vote)| vote),
6851 );
6852 if !matches!(
6863 round_record.e2e_status(),
6864 E2eStatus::Deferred | E2eStatus::NotConfigured
6865 ) {
6866 round_record.verified_head = Some(head.clone());
6867 round_record.verified_at = Some(Timestamp::now());
6868 }
6869
6870 if incomplete {
6871 let missing: Vec<String> = round_record
6872 .reviews
6873 .iter()
6874 .filter(|r| r.failed.is_some())
6875 .map(|r| format!("review-{}", r.reviewer))
6876 .collect();
6877 self.state.event(
6878 "review",
6879 format!(
6880 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6881 missing.join(", ")
6882 ),
6883 );
6884 }
6885
6886 if clean {
6887 self.state.event(
6888 "review",
6889 if incomplete && policy == IncompleteReviewPolicy::Warn {
6890 format!(
6891 "round {round}: clean (warn policy, incomplete panel) — no \
6892 blocking findings from the seats that answered, verification green"
6893 )
6894 } else if incomplete {
6895 format!(
6896 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6897 quorum) — no blocking findings from the seats that answered, \
6898 verification green",
6899 expected - answered
6900 )
6901 } else {
6902 format!("round {round}: clean — no blocking findings, verification green")
6903 },
6904 );
6905 self.state.reviews.push(round_record);
6906 self.state.status = RunStatus::Gating;
6907 self.state.save()?;
6908 return Ok(());
6909 }
6910
6911 if incomplete && blocking == 0 && e2e_ok {
6919 self.state.reviews.push(round_record);
6920 self.state.save()?;
6921 if round == max_rounds {
6922 self.state.status = RunStatus::Blocked;
6923 self.state.event(
6924 "review",
6925 format!(
6926 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6927 refusing to call it clean",
6928 expected - answered
6929 ),
6930 );
6931 return Ok(());
6932 }
6933 continue;
6934 }
6935
6936 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6948 self.state.reviews.push(round_record);
6949 return self
6950 .stop_unresolved_review(
6951 "the round's own verification could not run",
6952 &shell,
6953 &winner.worktree,
6954 )
6955 .await;
6956 }
6957
6958 if round == max_rounds {
6959 self.state.reviews.push(round_record);
6960 return self
6961 .stop_reviewing(
6962 &format!(
6963 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6964 ),
6965 &shell,
6966 &winner.worktree,
6967 )
6968 .await;
6969 }
6970
6971 if !self.fix_review_round(&winner, round_record).await? {
6972 return Ok(());
6973 }
6974 }
6975 Ok(())
6976 }
6977
6978 async fn fix_review_round(
6981 &mut self,
6982 winner: &Candidate,
6983 mut round_record: ReviewRound,
6984 ) -> Result<bool> {
6985 let round = round_record.round;
6986 let shell = self.state.config.shell();
6987 let step = self.run_fix_round(winner, &mut round_record).await?;
6988 if matches!(step, FixStep::Resource) {
6989 self.stop_for_resource()?;
6990 return Ok(false);
6991 }
6992 if self.state.reviews.len() < round {
6993 self.state.reviews.push(round_record);
6994 } else {
6995 self.state.reviews[round - 1] = round_record;
6996 }
6997 self.state.save()?;
6998
6999 if matches!(step, FixStep::Recorded { report_lost: true }) {
7012 self.stop_reviewing(
7013 "the fixer's adoption report never came back, even after resuming its \
7014 own seat; refusing to start another round against the same worktree \
7015 while that is unresolved",
7016 &shell,
7017 &winner.worktree,
7018 )
7019 .await?;
7020 return Ok(false);
7021 }
7022
7023 let streak = self
7024 .state
7025 .reviews
7026 .iter()
7027 .rev()
7028 .take_while(|r| !r.progressed)
7029 .count();
7030 if streak >= STAGNANT_LIMIT {
7031 self.stop_reviewing(
7032 &format!("the tree has not moved against base for {streak} round(s) in a row"),
7033 &shell,
7034 &winner.worktree,
7035 )
7036 .await?;
7037 return Ok(false);
7038 }
7039 Ok(true)
7040 }
7041
7042 async fn run_fix_round(
7044 &mut self,
7045 winner: &Candidate,
7046 round_record: &mut ReviewRound,
7047 ) -> Result<FixStep> {
7048 let round = round_record.round;
7049 let max_rounds = self.state.config.graph.review_rounds;
7050 let run_id = self.state.id.clone();
7051 let prompts = self.state.config.prompts.clone();
7052 let language = self.state.config.graph.language.clone();
7053 let sessions = self.state.config.graph.sessions;
7054 let artifacts = self.merge_repair_artifacts();
7055 let base = self.landing_base();
7056 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7057 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7058 let this_round_verification = round_record.verification_summary(&head);
7059 let all_findings: Vec<_> = round_record
7060 .reviews
7061 .iter()
7062 .flat_map(|r| r.findings.iter().cloned())
7063 .collect();
7064 let blocking_findings: Vec<_> = all_findings
7067 .iter()
7068 .filter(|f| f.severity.blocks())
7069 .cloned()
7070 .collect();
7071 let fix_prompt = prompt::fix_in(
7072 &self.state.instruction,
7073 &blocking_findings,
7074 this_round_verification.as_ref(),
7075 round,
7076 max_rounds,
7077 &language,
7078 &self.state.config.graph.github_language,
7079 );
7080 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7081 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7082 let (job, seat, out) = self
7083 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7084 prompt: fix_prompt.clone(),
7085 spec,
7086 seat,
7087 cwd: winner.worktree.clone(),
7088 timeout,
7089 allow_write: true,
7090 sessions,
7091 artifacts: artifacts.clone(),
7092 stem: format!("fix-{round}"),
7093 handover: None,
7094 })
7095 .await;
7096 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7099 return Ok(FixStep::Resource);
7100 }
7101 let agent_id = seat.agent.clone();
7102
7103 let mut fix = FixRecord {
7104 agent: agent_id,
7105 addressed: Vec::new(),
7106 rejected: Vec::new(),
7107 notes: String::new(),
7108 committed: false,
7109 failed: None,
7110 duration_ms: 0,
7111 continuation: None,
7112 };
7113 let mut continuation = ContinuationRecord::not_needed();
7114 let mut final_seat = seat.clone();
7115 match out {
7116 AgentOutcome::Ok(o) => {
7117 fix.duration_ms = o.duration_ms;
7118 let parsed = verdict::extract_json::<FixReport>(&o.text);
7119 let incomplete_reason = match &parsed {
7126 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7127 "the reply parsed, but it reported a command whose own CLI never \
7128 confirmed an exit status"
7129 .to_owned(),
7130 ),
7131 Ok(_) => None,
7132 Err(e) => Some(e.to_string()),
7133 };
7134 match incomplete_reason {
7135 None => {
7136 let report = parsed.expect("checked Ok above");
7137 fix.addressed = report.addressed;
7138 fix.rejected = report.rejected;
7139 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7140 }
7141 Some(reason) => {
7142 let (resumed_seat, resolved, failure, cont) = self
7143 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7144 .await;
7145 fix.duration_ms += cont.cumulative_wait_ms;
7146 continuation = cont;
7147 final_seat = resumed_seat;
7148 match resolved {
7149 Some(report) => {
7150 fix.addressed = report.addressed;
7151 fix.rejected = report.rejected;
7152 fix.notes =
7153 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7154 }
7155 None => fix.failed = failure,
7156 }
7157 }
7158 }
7159 }
7160 AgentOutcome::Dropped(o) => {
7162 fix.duration_ms = o.duration_ms;
7163 let why = o
7164 .dropped
7165 .as_ref()
7166 .map(|d| d.why.as_str())
7167 .unwrap_or("the CLI ended the stream without delivering its answer");
7168 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7169 }
7170 AgentOutcome::Quota(o) => {
7171 self.state.quota.push(QuotaLoss::from_report(
7172 final_seat.key.clone(),
7173 "fix",
7174 o.quota.as_ref(),
7175 ));
7176 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7177 }
7178 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7179 }
7180 if self.state.resource_block.is_some() {
7181 self.state.seats.insert(final_seat.key.clone(), final_seat);
7182 return Ok(FixStep::Resource);
7183 }
7184 fix.continuation = Some(continuation);
7185 self.state.seats.insert(final_seat.key.clone(), final_seat);
7186 let coordination = self.coordinate(&winner.worktree).await?;
7187 if let Ok(r) = git::rescue_commit(
7188 &winner.worktree,
7189 &format!("magi: review round {round} fixes (uncommitted work)"),
7190 )
7191 .await
7192 {
7193 self.state.note_withheld("fix", &r.withheld);
7194 }
7195 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7196 fix.committed = after != before;
7197 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7205 drop(coordination);
7206 let progressed = diff_after != patch;
7207 let commit_note = if fix.committed {
7208 "committed"
7209 } else {
7210 "NO new commit"
7211 };
7212 let tree_note = if progressed {
7213 "changed vs base"
7214 } else {
7215 "unchanged vs base"
7216 };
7217 self.state.event(
7218 "fix",
7219 match &fix.failed {
7220 Some(reason) => {
7226 format!(
7227 "round {round}: fixer's adoption report was lost ({reason}); \
7228 {commit_note}, tree {tree_note}"
7229 )
7230 }
7231 None => format!(
7232 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7233 {tree_note}{}",
7234 fix.addressed.len(),
7235 fix.rejected.len(),
7236 if continuation.outcome == ContinuationOutcome::Resumed {
7237 format!(
7238 " (adoption report recovered after {} continuation(s))",
7239 continuation.attempts
7240 )
7241 } else {
7242 String::new()
7243 },
7244 ),
7245 },
7246 );
7247 round_record.fix = Some(fix);
7248 round_record.progressed = progressed;
7249 Ok(FixStep::Recorded {
7250 report_lost: matches!(
7251 continuation.outcome,
7252 ContinuationOutcome::Exhausted
7253 | ContinuationOutcome::QuotaLost
7254 | ContinuationOutcome::NoSession
7255 ),
7256 })
7257 }
7258
7259 fn record_contested_handoff(&mut self) {
7293 if self.state.contested_handoff.is_some() {
7294 return;
7295 }
7296 let Some(contested) = self
7297 .state
7298 .reviews
7299 .last()
7300 .and_then(ReviewRound::contested_handoff)
7301 else {
7302 return;
7303 };
7304 self.state.event(
7305 "review",
7306 format!(
7307 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7308 wait for the owner's approval",
7309 contested.findings.len(),
7310 contested.rejecters.len()
7311 ),
7312 );
7313 self.state.contested_handoff = Some(contested);
7314 }
7315
7316 async fn stop_unresolved_review(
7317 &mut self,
7318 why: &str,
7319 shell: &[String],
7320 worktree: &Path,
7321 ) -> Result<()> {
7322 self.stop_reviewing_inner(why, shell, worktree, true).await
7323 }
7324
7325 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7326 self.stop_reviewing_inner(why, shell, worktree, false).await
7327 }
7328
7329 async fn stop_reviewing_inner(
7330 &mut self,
7331 why: &str,
7332 shell: &[String],
7333 worktree: &Path,
7334 repairable: bool,
7335 ) -> Result<()> {
7336 let round_idx = self.state.reviews.len() - 1;
7337 let needs_catchup_run = matches!(
7345 self.state.reviews[round_idx].e2e_status(),
7346 E2eStatus::Deferred | E2eStatus::ResourceBlocked
7347 );
7348 if needs_catchup_run {
7349 let round = self.state.reviews[round_idx].round;
7350 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7351 let commands = self.state.config.verify.e2e.clone();
7352 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7353 let cache_dir = self.state.config.cache_dir();
7354 let context = format!(
7355 "round {round}: verification unresolved, catching up before the final decision"
7356 );
7357 let wait = self.repository_wait();
7358 let (outcomes, verify_retried) = with_cache_lease(
7359 &mut self.state,
7360 cache_dir.as_deref(),
7361 "e2e",
7362 "e2e",
7363 worktree,
7364 &attempted_head,
7365 timeout,
7366 &context,
7367 &wait,
7368 |state, budget| {
7369 let shell = shell.to_vec();
7370 let commands = commands.clone();
7371 let context = context.clone();
7372 async move {
7373 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7374 .await
7375 }
7376 },
7377 )
7378 .await?;
7379 let last = &mut self.state.reviews[round_idx];
7380 last.e2e = outcomes;
7381 last.verify_retried = verify_retried;
7382 last.verified_head = Some(attempted_head);
7389 last.verified_at = Some(Timestamp::now());
7390 if verify_inconclusive(&last.e2e) {
7391 self.state.save()?;
7398 return Ok(());
7399 }
7400 last.e2e_deferred = false;
7401 }
7402 let last = &self.state.reviews[round_idx];
7403 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7404
7405 match last.e2e_status() {
7406 E2eStatus::Failed => {
7407 let red: Vec<String> = last
7408 .e2e
7409 .iter()
7410 .filter(|o| !o.ok())
7411 .map(|o| {
7412 format!(
7413 "`{}` -> {:?}\n{}",
7414 o.command,
7415 o.code,
7416 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7417 )
7418 })
7419 .collect();
7420 let can_repair = last.blocking == 0 && last.fix.is_none();
7421 self.state.reviews[round_idx].clean = false;
7422 self.state
7423 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7424 self.state.status = RunStatus::Blocked;
7425 if repairable
7426 && can_repair
7427 && self.state.reviews.len() < self.state.config.graph.review_rounds
7428 {
7429 self.state.status = RunStatus::Reviewing;
7430 }
7431 }
7432 E2eStatus::ResourceBlocked => {
7437 self.state.event(
7438 "review",
7439 format!(
7440 "{why}; e2e could not run (resource unavailable); not \
7441 deciding yet"
7442 ),
7443 );
7444 }
7445 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7446 self.state.event(
7447 "review",
7448 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7449 );
7450 self.record_contested_handoff();
7451 self.state.status = RunStatus::Gating;
7452 }
7453 }
7454 self.state.save()?;
7455 Ok(())
7456 }
7457
7458 async fn gate(&mut self) -> Result<()> {
7461 if self.hold_base_sync().await? {
7462 return Ok(());
7463 }
7464 if self.state.status == RunStatus::Failed
7476 || self
7477 .state
7478 .base_sync
7479 .as_ref()
7480 .is_some_and(|s| s.conflict.is_some())
7481 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7482 != Some(RunStatus::Gating)
7483 {
7484 return Ok(());
7485 }
7486 let Some(winner) = self.state.winner().cloned() else {
7487 return Ok(());
7488 };
7489 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7490 self.state.reconcile_verification_head(&head);
7491 self.state.save()?;
7492 let wait = self.repository_wait();
7493 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7494 return Ok(());
7495 }
7496 if self.state.gate_ran {
7497 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7508 self.state.status = RunStatus::Blocked;
7509 self.state.save()?;
7510 }
7511 return Ok(());
7512 }
7513 self.state.status = RunStatus::Gating;
7514 let mut outcomes = self.run_gate(&winner).await?;
7515 let mut attempted_head = self.state.gate_head.clone();
7516 loop {
7517 if verify_inconclusive(&outcomes) {
7528 self.state.save()?;
7529 return Ok(());
7530 }
7531 if outcomes.iter().all(CommandOutcome::ok) {
7532 break;
7533 }
7534 let fixes_before = self.state.gate_fixes.len();
7535 match self.gate_fix_round(&winner, &outcomes).await? {
7536 GateFix::Retry => {
7537 outcomes = self.run_gate(&winner).await?;
7538 attempted_head = self.state.gate_head.clone();
7539 }
7540 GateFix::Stop => {
7541 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7542 if self.state.gate_fixes.len() > fixes_before
7543 && attempted_head.as_deref() != Some(current.as_str())
7544 && verify_current_head(&mut self.state, &winner, &wait).await?
7545 {
7546 outcomes = self.run_gate(&winner).await?;
7549 attempted_head = self.state.gate_head.clone();
7550 } else {
7551 break;
7552 }
7553 }
7554 GateFix::Defer => {
7555 self.state.save()?;
7556 return Ok(());
7557 }
7558 }
7559 }
7560 let passed = outcomes.iter().all(CommandOutcome::ok);
7561 self.state.gate = outcomes;
7562 self.state.gate_ran = true;
7563 self.state.gate_head = attempted_head;
7564 self.state.save()?;
7565 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7566 return Ok(());
7567 }
7568 if !passed {
7569 self.state.status = RunStatus::Blocked;
7570 let spent = self.state.gate_fixes.len();
7571 self.state.event(
7572 "gate",
7573 if spent == 0 {
7574 "gate failed; not merging".to_owned()
7575 } else {
7576 format!("gate failed after {spent} gate-fix round(s); not merging")
7577 },
7578 );
7579 }
7580 self.state.save()?;
7581 Ok(())
7582 }
7583
7584 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7594 let commands = state.config.verify.pre_gate.clone();
7595 if commands.is_empty() {
7596 return;
7597 }
7598 let shell = state.config.shell();
7599 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7600 let (outcomes, _) = run_commands(
7601 state,
7602 "pre_gate",
7603 "pre_gate",
7604 0,
7605 &shell,
7606 &commands,
7607 &winner.worktree,
7608 timeout,
7609 )
7610 .await;
7611 for o in &outcomes {
7612 if !o.ok() {
7613 tracing::warn!(
7614 "pre_gate `{}` failed ({:?}); the gate decides",
7615 o.command,
7616 o.code
7617 );
7618 }
7619 state.event(
7620 "pre_gate",
7621 format!(
7622 "`{}` -> {}",
7623 o.command,
7624 if o.ok() {
7625 "pass".to_owned()
7626 } else {
7627 format!(
7628 "FAIL ({:?})\n{}",
7629 o.code,
7630 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7631 )
7632 }
7633 ),
7634 );
7635 }
7636 state.pre_gate = outcomes;
7637 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7638 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7639 Ok(head) => {
7640 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7641 state.pre_gate_commit = Some(head);
7642 }
7643 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7644 },
7645 Ok(false) => {}
7646 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7647 }
7648 if let Err(e) = state.save() {
7649 tracing::warn!("could not persist the pre_gate record: {e:#}");
7650 }
7651 }
7652
7653 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7656 let wait = self.repository_wait();
7657 run_landing_gate(&mut self.state, winner, &wait).await
7658 }
7659
7660 async fn gate_fix_round(
7673 &mut self,
7674 winner: &Candidate,
7675 outcomes: &[CommandOutcome],
7676 ) -> Result<GateFix> {
7677 let cap = self.state.config.graph.gate_fix_rounds;
7678 let spent = self.state.gate_fixes.len();
7679 if spent >= cap {
7680 if cap > 0 {
7681 self.state.event(
7682 "gate",
7683 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7684 );
7685 }
7686 return Ok(GateFix::Stop);
7687 }
7688 if !gate_fixable(outcomes) {
7689 self.state.event(
7690 "gate",
7691 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7692 command or similar); not spending a fix round on it",
7693 );
7694 return Ok(GateFix::Stop);
7695 }
7696 let min_free = self.state.config.disk.min_free_bytes;
7697 if min_free > 0 {
7698 match crate::disk::free_bytes(&winner.worktree) {
7699 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7700 Ok(free) => {
7701 self.state.event(
7702 "gate",
7703 format!(
7704 "only {free} bytes free ({min_free} required by `[disk] \
7705 min_free_bytes`); not spending a fix round on a failure the disk \
7706 may explain"
7707 ),
7708 );
7709 return Ok(GateFix::Stop);
7710 }
7711 Err(e) => {
7712 self.state.event(
7713 "gate",
7714 format!("free disk space could not be measured ({e:#}); no fix round"),
7715 );
7716 return Ok(GateFix::Stop);
7717 }
7718 }
7719 }
7720
7721 let attempt = spent + 1;
7722 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7723 let base = self.landing_base();
7724 let fix_prompt = prompt::gate_fix_in(
7725 &self.state.instruction,
7726 &failed,
7727 attempt,
7728 cap,
7729 &self.state.config.graph.language,
7730 &self.state.config.graph.github_language,
7731 );
7732 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7733 let sessions = self.state.config.graph.sessions;
7734 let artifacts = self.merge_repair_artifacts();
7735 self.state.event(
7736 "gate",
7737 format!("gate failed; gate-fix round {attempt} of {cap}"),
7738 );
7739 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7740 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7741 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7744 head: before.clone(),
7745 passed: false,
7746 outcomes: Vec::new(),
7747 });
7748 self.state.save()?;
7749 let (_, seat, out) = self
7750 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7751 prompt: fix_prompt.clone(),
7752 spec,
7753 seat,
7754 cwd: winner.worktree.clone(),
7755 timeout,
7756 allow_write: true,
7757 sessions,
7758 artifacts: artifacts.clone(),
7759 stem: format!("gate-fix-{attempt}"),
7760 handover: None,
7761 })
7762 .await;
7763 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7766 return Ok(GateFix::Defer);
7767 }
7768 let mut record = GateFixRecord {
7769 agent: seat.agent.clone(),
7770 failed,
7771 notes: String::new(),
7772 committed: false,
7773 error: None,
7774 };
7775 match out {
7776 AgentOutcome::Ok(o) => {
7777 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7780 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7781 }
7782 }
7783 AgentOutcome::Dropped(_) => {
7784 record.error = Some("the CLI dropped the stream".to_owned());
7785 }
7786 AgentOutcome::Quota(o) => {
7787 self.state.quota.push(QuotaLoss::from_report(
7788 seat.key.clone(),
7789 "gate-fix",
7790 o.quota.as_ref(),
7791 ));
7792 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7793 }
7794 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7795 }
7796 self.state.seats.insert(seat.key.clone(), seat);
7797 let coordination = self.coordinate(&winner.worktree).await?;
7798 if let Ok(r) = git::rescue_commit(
7799 &winner.worktree,
7800 &format!("magi: gate fix {attempt} (uncommitted work)"),
7801 )
7802 .await
7803 {
7804 self.state.note_withheld("gate-fix", &r.withheld);
7805 }
7806 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7807 record.committed = after != before;
7808 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7809 let note = record.error.clone();
7810 self.state.gate_fixes.push(record);
7811 self.state.save()?;
7812 if !changed {
7813 self.state.event(
7814 "gate",
7815 match note {
7816 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7817 None => format!("gate-fix round {attempt}: the tree did not change"),
7818 },
7819 );
7820 return Ok(GateFix::Stop);
7821 }
7822 self.state.event(
7823 "gate",
7824 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7825 );
7826
7827 drop(coordination);
7828 let wait = self.repository_wait();
7829 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7830 }
7831
7832 async fn guarded_pr_message(
7839 &mut self,
7840 winner: &Candidate,
7841 facts: Option<&BranchFacts>,
7842 posting: bool,
7843 ) -> Result<Option<PrMessage>> {
7844 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7845 let head = match final_head.as_deref() {
7846 Some(h) => crate::pr_outcome::FinalHead::At(h),
7847 None => crate::pr_outcome::FinalHead::Unknown,
7848 };
7849 let summarized = if posting {
7851 Some(self.summarize_change(winner, final_head.as_deref()).await)
7852 } else {
7853 None
7854 };
7855 if self.state.resource_block.is_some() {
7860 self.state.event(
7861 "pr-summary",
7862 "a shared resource was busy; nothing was published, resume to summarize again",
7863 );
7864 return Ok(None);
7865 }
7866 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7867 anyhow::bail!(
7868 "the final commit moved during public description generation; retry before publishing"
7869 );
7870 }
7871 let note = match (&summarized, &self.state.change_summary) {
7872 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7873 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7874 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7875 (None, _) => ChangeNote::NotAsked,
7876 };
7877 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7878 if !posting {
7879 let identity = crate::scrub::Identity::current();
7880 return Ok(Some(PrMessage {
7881 title: crate::scrub::scrub(&pr.title, &identity),
7882 body: crate::scrub::scrub(&pr.body, &identity),
7883 }));
7884 }
7885 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7888 if let Some(known) = self
7889 .state
7890 .github_text
7891 .clone()
7892 .filter(|g| g.fingerprint == raw_fp)
7893 {
7894 return self.settle_withheld(winner, &pr, known).await;
7895 }
7896 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7897 let mut violations =
7898 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7899 let identity = crate::scrub::Identity::current();
7900 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7901 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7902 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7903 {
7904 violations.push(crate::github_text::Violation::SensitiveData);
7905 }
7906 let has_language = violations
7910 .iter()
7911 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7912 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7913 self.state.event(
7914 "github-text",
7915 if has_language {
7916 format!("description rejected: {violations:?}; requesting one rewrite")
7917 } else {
7918 format!("description rejected: {violations:?}; asking the owner")
7919 },
7920 );
7921 let mut rewritten = false;
7922 let mut rejected_rewrite = None;
7923 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7924 {
7925 let output = text;
7926 if let Some(title) = summary_title(&output) {
7927 let raw_body = summary_without_title(&output);
7930 let clean_title = crate::scrub::scrub(&title, &identity);
7931 let mut body = crate::scrub::scrub(&raw_body, &identity);
7932 let redacted = clean_title != title || body != raw_body;
7933 let title = clean_title;
7934 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7935 if !complete {
7936 self.state.event(
7937 "github-text",
7938 "description rewrite omitted required findings",
7939 );
7940 }
7941 if !body.trim().is_empty() {
7942 body.push_str("\n\n");
7943 body.push_str(&public_pr_evidence(&self.state, head));
7944 }
7945 let marker = format!("magi:run/{}", self.state.id);
7946 if !body.trim().is_empty() && !body.contains(&marker) {
7947 body.push_str(&format!("\n\n{marker}"));
7948 }
7949 let rewrite = PrMessage {
7950 title: title.clone(),
7951 body: body.clone(),
7952 };
7953 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7954 let mut failed = crate::github_text::check_for(
7955 &self.state.config.graph,
7956 &title,
7957 &body,
7958 redecision,
7959 );
7960 if (!crate::github_text::shareable(&title)
7961 || !crate::github_text::shareable(&body))
7962 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7963 {
7964 failed.push(crate::github_text::Violation::SensitiveData);
7965 }
7966 if complete
7967 && !title.trim().is_empty()
7968 && !body.trim().is_empty()
7969 && !failed.is_empty()
7970 {
7971 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7972 title: title.clone(),
7973 body: body.clone(),
7974 failed: failed
7975 .iter()
7976 .map(|v| crate::github_text::category(*v).to_owned())
7977 .collect(),
7978 });
7979 }
7980 if complete
7981 && !title.trim().is_empty()
7982 && !raw_body.trim().is_empty()
7983 && failed.is_empty()
7984 {
7985 decision = redecision;
7986 pr = PrMessage { title, body };
7987 rewritten = true;
7988 self.state
7989 .event("github-text", "description rewrite passed");
7990 if redacted {
7991 self.state.event(
7992 "github-text",
7993 "sensitive data removed from description rewrite",
7994 );
7995 }
7996 }
7997 }
7998 }
7999 if !rewritten {
8000 self.state.event(
8001 "github-text",
8002 "description rewrite unavailable or rejected; asking the owner",
8003 );
8004 if let Some(w) =
8005 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8006 {
8007 self.retire_github_text_question("the withheld text changed");
8010 let record = crate::run::GithubTextAsk {
8011 rewrite: rejected_rewrite,
8012 fingerprint: raw_fp,
8013 title: w.title,
8014 body: w.body,
8015 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8016 question: None,
8017 asks: 0,
8018 resolved: false,
8019 chosen_title: None,
8020 choice: None,
8021 };
8022 return self.settle_withheld(winner, &pr, record).await;
8023 }
8024 }
8025 }
8026 let (title, body) =
8027 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8028 Ok(Some(PrMessage { title, body }))
8029 }
8030
8031 async fn summarize_change(
8042 &mut self,
8043 winner: &Candidate,
8044 head: Option<&str>,
8045 ) -> std::result::Result<(), String> {
8046 let Some(head) = head else {
8047 return Err("the final commit could not be read".to_owned());
8048 };
8049 let remote = self.state.config.merge.remote.clone();
8050 let tracking = format!("{remote}/{}", self.state.base_branch);
8051 if !git::rev_exists(&winner.worktree, &tracking).await {
8052 return Err(format!("the remote base `{tracking}` could not be read"));
8053 }
8054 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8055 Ok(b) if !b.is_empty() => b,
8056 _ => return Err("the merge base with the remote base could not be read".to_owned()),
8057 };
8058 let evidence = crate::pr_outcome::summary_evidence(
8059 &self.state,
8060 crate::pr_outcome::FinalHead::At(head),
8061 );
8062 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8063 if self
8064 .state
8065 .change_summary
8066 .as_ref()
8067 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8068 {
8069 return Ok(());
8070 }
8071 if self.state.change_summary.take().is_some() {
8072 self.state.event(
8073 "pr-summary",
8074 "the final diff or evidence changed; the earlier change summary was discarded",
8075 );
8076 }
8077 if let Err(e) = write_artifact(
8079 &self.state,
8080 &format!("public-description-{fingerprint}.md"),
8081 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8082 ) {
8083 tracing::warn!("could not save public description evidence: {e:#}");
8084 }
8085 let chain = match agent::pick_chain(
8086 &self.state.config.agents,
8087 self.state.config.roles.synthesizer.as_ref(),
8088 &agent::installed,
8089 "synthesizer",
8090 ) {
8091 Ok(chain) => chain,
8092 Err(e) => {
8093 tracing::warn!("no agent for the change summary: {e:#}");
8094 self.state.event("pr-summary", "no summary agent available");
8095 return Err("no agent was available to summarize it".to_owned());
8096 }
8097 };
8098 let subjects = git::subjects(&winner.worktree, &base, head)
8099 .await
8100 .unwrap_or_default();
8101 let stat = git::diff_stat(&winner.worktree, &base, head)
8102 .await
8103 .unwrap_or_default();
8104 let stat: String = stat.chars().take(6_000).collect();
8105 let mut prompt = crate::prompt::change_summary_in(
8106 &base,
8107 head,
8108 &subjects,
8109 stat.trim(),
8110 &self.state.config.graph.github_language,
8111 );
8112 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8113 let sessions = self.state.config.graph.sessions;
8114 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8115 let prompts = self.state.config.prompts.clone();
8116 let cache = self.state.config.cache_dir();
8117 let run = self.state.id.clone();
8118 for (i, spec) in chain.into_iter().enumerate() {
8119 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8120 let github_spec = match agent::for_github_text(
8121 &spec,
8122 &self.state.config.graph.github_language,
8123 &winner.worktree,
8124 ) {
8125 Ok(spec) => spec,
8126 Err(e) => {
8127 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8128 continue;
8129 }
8130 };
8131 let job = SeatJob {
8132 spec: github_spec,
8133 seat,
8134 cwd: winner.worktree.clone(),
8135 prompt: prompt.clone(),
8136 timeout,
8137 allow_write: false,
8138 sessions,
8139 artifacts: agent::artifacts_dir(&self.state.dir()),
8140 stem: if i == 0 {
8141 "change-summary".to_owned()
8142 } else {
8143 format!("change-summary-{}", spec.id)
8144 },
8145 handover: None,
8146 };
8147 let ctx = WaveCtx {
8148 run: &run,
8149 node: "pr-summary",
8150 prompts: &prompts,
8151 cache: cache.as_deref(),
8152 round: None,
8153 carry_seats: false,
8154 pauses: Some((&self.pause, &self.interrupt)),
8155 };
8156 let (seat, outcome) =
8157 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8158 self.state.seats.insert(seat.key.clone(), seat);
8159 let text = match outcome {
8160 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8161 AgentOutcome::ResourceBlocked(why) => {
8162 if self.state.resource_block.is_none() {
8165 self.state.resource_block = Some(crate::run::ResourceBlock {
8166 cause: crate::run::ResourceBlockCause::Other,
8167 node: "pr-summary".to_owned(),
8168 kind: "shared resource".to_owned(),
8169 detail: why,
8170 at: Timestamp::now(),
8171 });
8172 }
8173 return Err("a shared resource was busy".to_owned());
8174 }
8175 _ => continue,
8176 };
8177 let blind = &self.state.config.blind;
8178 let Some(title) = summary_title(&text) else {
8179 continue;
8180 };
8181 let body: String = summary_without_title(&text)
8182 .lines()
8183 .filter(|l| !l.contains("magi:run/"))
8184 .collect::<Vec<_>>()
8185 .join("\n");
8186 let body = blind::sanitize_prose(body.trim(), blind);
8187 let title = blind::sanitize_prose(&title, blind);
8188 if body.trim().is_empty() || title.trim().is_empty() {
8189 continue;
8190 }
8191 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8192 continue;
8193 }
8194 if git::rev_parse(&winner.worktree, "HEAD")
8196 .await
8197 .ok()
8198 .as_deref()
8199 != Some(head)
8200 {
8201 return Err("the branch moved while the summary was written".to_owned());
8202 }
8203 if git::merge_base(&winner.worktree, &tracking, head)
8204 .await
8205 .ok()
8206 .as_deref()
8207 != Some(base.as_str())
8208 || crate::pr_outcome::summary_evidence(
8209 &self.state,
8210 crate::pr_outcome::FinalHead::At(head),
8211 ) != evidence
8212 {
8213 return Err("the summary evidence changed while it was written".to_owned());
8214 }
8215 self.state.change_summary = Some(crate::run::ChangeSummary {
8216 head: head.to_owned(),
8217 base,
8218 evidence: fingerprint,
8219 title,
8220 body,
8221 });
8222 self.state.event(
8223 "pr-summary",
8224 format!("change summary written by {} for the final diff", spec.id),
8225 );
8226 if let Err(e) = self.state.save() {
8227 tracing::warn!("could not save the change summary: {e:#}");
8228 }
8229 return Ok(());
8230 }
8231 self.state
8232 .event("pr-summary", "no agent produced a change summary");
8233 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8234 }
8235
8236 async fn rewrite_pr_text(
8246 &mut self,
8247 winner: &Candidate,
8248 pr: &PrMessage,
8249 violations: &[crate::github_text::Violation],
8250 ) -> Option<String> {
8251 let prompt = format!(
8252 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8253 pr.title, pr.body
8254 );
8255 let sessions = self.state.config.graph.sessions;
8256 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8257 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8260 let key = format!("impl-{}", winner.label);
8261 if let Some(spec) = self
8262 .state
8263 .config
8264 .agents
8265 .iter()
8266 .find(|a| a.id == winner.agent)
8267 .cloned()
8268 && let Some(existing) = self.state.seats.get(&key)
8269 && existing.agent == spec.id
8270 && agent::has_session(spec.kind, existing, sessions)
8271 {
8272 attempts.push((spec, Some(existing.clone())));
8273 }
8274 match agent::pick_chain(
8275 &self.state.config.agents,
8276 self.state.config.roles.language_judge.as_ref(),
8277 &agent::installed,
8278 "language_judge",
8279 ) {
8280 Ok(chain) => {
8281 for spec in chain {
8282 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8283 continue;
8284 }
8285 attempts.push((spec, None));
8286 }
8287 }
8288 Err(e) if attempts.is_empty() => {
8289 tracing::warn!("no agent for the description rewrite: {e:#}");
8290 self.state
8291 .event("github-text", "no rewrite agent available");
8292 return None;
8293 }
8294 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8295 }
8296 let prompts = self.state.config.prompts.clone();
8297 let cache = self.state.config.cache_dir();
8298 let run = self.state.id.clone();
8299 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8300 let resumed = resumed_seat.is_some();
8301 let seat = resumed_seat.unwrap_or_else(|| {
8302 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8303 });
8304 let github_spec = match agent::for_github_text(
8305 &spec,
8306 &self.state.config.graph.github_language,
8307 &winner.worktree,
8308 ) {
8309 Ok(spec) => spec,
8310 Err(e) => {
8311 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8312 continue;
8313 }
8314 };
8315 let job = SeatJob {
8316 spec: github_spec,
8317 seat,
8318 cwd: winner.worktree.clone(),
8319 prompt: prompt.clone(),
8320 timeout: retry_budget(timeout_full, resumed),
8321 allow_write: false,
8322 sessions,
8323 artifacts: agent::artifacts_dir(&self.state.dir()),
8324 stem: if i == 0 {
8325 "github-text-rewrite".to_owned()
8326 } else {
8327 format!("github-text-rewrite-{}", spec.id)
8328 },
8329 handover: None,
8330 };
8331 let ctx = WaveCtx {
8332 run: &run,
8333 node: "github-text",
8334 prompts: &prompts,
8335 cache: cache.as_deref(),
8336 round: None,
8337 carry_seats: false,
8338 pauses: Some((&self.pause, &self.interrupt)),
8339 };
8340 let (seat, outcome) =
8341 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8342 self.state.seats.insert(seat.key.clone(), seat);
8343 match outcome {
8344 AgentOutcome::Ok(o)
8345 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8346 {
8347 return Some(o.text);
8348 }
8349 AgentOutcome::ResourceBlocked(_) => return None,
8350 _ => {}
8351 }
8352 }
8353 None
8354 }
8355
8356 fn retire_github_text_question(&mut self, why: &str) {
8358 let Some(id) = self
8359 .state
8360 .github_text
8361 .as_ref()
8362 .and_then(|g| g.question.clone())
8363 else {
8364 return;
8365 };
8366 let _ = ask::Questions::open().update(&id, |q| {
8367 q.abandon(why);
8368 Ok(())
8369 });
8370 }
8371
8372 async fn settle_withheld(
8376 &mut self,
8377 winner: &Candidate,
8378 pr: &PrMessage,
8379 mut g: crate::run::GithubTextAsk,
8380 ) -> Result<Option<PrMessage>> {
8381 use crate::github_text as gt;
8382 let store = ask::Questions::open();
8383 let timeout = self.state.config.graph.answer_timeout;
8384 if !g.resolved {
8385 let mut standing = g
8386 .question
8387 .as_ref()
8388 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8389 if let Some(q) = standing.as_ref()
8390 && q.status.open()
8391 && gt::expired(q, timeout)
8392 {
8393 let why = format!("no answer within {}s of asking", timeout.max(1));
8394 standing = store
8395 .update(&q.id, |q| {
8396 q.abandon(&why);
8397 Ok(())
8398 })
8399 .ok()
8400 .map(|(q, ())| q);
8401 }
8402 let mut ask_again: Option<Option<Vec<String>>> = None;
8403 match standing {
8404 None if g.asks <= 1 => {
8407 g.asks = 0;
8408 ask_again = Some(None);
8409 }
8410 None => {
8411 g.resolved = true;
8412 g.chosen_title = None;
8413 g.choice = Some(GithubTextChoice::Fallback);
8414 }
8415 Some(q) if q.status.open() => {
8416 self.park_on_github_text(&g)?;
8417 return Ok(None);
8418 }
8419 Some(q) => match gt::read_reply(&q) {
8420 gt::Reply::Fallback => {
8421 g.resolved = true;
8422 g.chosen_title = None;
8423 g.choice = Some(GithubTextChoice::Fallback);
8424 }
8425 gt::Reply::Original => {
8426 g.resolved = true;
8427 g.chosen_title = None;
8428 g.choice = Some(GithubTextChoice::Original);
8429 }
8430 gt::Reply::Rewrite => {
8431 g.resolved = true;
8432 g.chosen_title = None;
8433 g.choice = Some(if g.rewrite.is_some() {
8434 GithubTextChoice::Rewrite
8435 } else {
8436 GithubTextChoice::Fallback
8437 });
8438 }
8439 gt::Reply::Title(text) => {
8440 let first = text
8441 .lines()
8442 .map(str::trim)
8443 .find(|l| !l.is_empty())
8444 .unwrap_or("");
8445 let decision = if first.is_empty() {
8446 None
8447 } else {
8448 let candidate = PrMessage {
8449 title: first.to_owned(),
8450 body: String::new(),
8451 };
8452 self.judge_pr_language(&winner.worktree, &candidate).await
8453 };
8454 match gt::vet_title(
8455 &text,
8456 decision,
8457 self.state.config.graph.github_english(),
8458 ) {
8459 Ok(title) => {
8460 g.resolved = true;
8461 g.chosen_title = Some(title);
8462 g.choice = Some(GithubTextChoice::MyTitle);
8463 }
8464 Err(cats) if g.asks < 2 => {
8465 ask_again = Some(Some(
8466 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8467 ))
8468 }
8469 Err(_) => {
8470 g.resolved = true;
8471 g.chosen_title = None;
8472 g.choice = Some(GithubTextChoice::Fallback);
8473 }
8474 }
8475 }
8476 },
8477 }
8478 if let Some(rejected) = ask_again {
8479 let w = gt::Withheld {
8482 title: g.title,
8483 body: g.body,
8484 categories: g.categories.clone(),
8485 };
8486 let artifact = match crate::run::write_artifact(
8489 &self.state,
8490 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8491 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8492 ) {
8493 Ok(p) => Some(p),
8494 Err(e) => {
8495 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8496 None
8497 }
8498 };
8499 let (detail, comparison) = gt::question_content(
8500 &w,
8501 &pr.title,
8502 &pr.body,
8503 &self.state.id,
8504 rejected.as_deref(),
8505 artifact.as_deref(),
8506 g.rewrite.as_ref(),
8507 );
8508 let mut q = ask::Question::new(
8509 self.state.id.clone(),
8510 gt::ASK_NODE.to_owned(),
8511 gt::ASK_SEAT.to_owned(),
8512 gt::question_summary(&self.state.config.graph.language, &w),
8513 detail,
8514 gt::question_choices(&w, g.rewrite.as_ref()),
8515 );
8516 q.posting_comparison = Some(comparison);
8517 q.answer_timeout = timeout;
8518 g.question = Some(q.id.clone());
8521 g.asks += 1;
8522 self.state.github_text = Some(g.clone());
8523 self.state.event(
8524 "github-text",
8525 format!("asking the owner about the withheld text ({})", q.short()),
8526 );
8527 self.state.save()?;
8528 store
8529 .put_executable(&mut q)
8530 .context("file the posting-gate question")?;
8531 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8532 tracing::warn!(
8533 "could not notify about posting-gate question {}: {e:#}",
8534 q.short()
8535 );
8536 }
8537 self.park_on_github_text(&g)?;
8538 return Ok(None);
8539 }
8540 }
8541 let resolved = g.chosen_title.clone();
8542 let original = g.choice == Some(GithubTextChoice::Original);
8543 self.state.event(
8545 "github-text",
8546 if original {
8547 format!(
8548 "posting the original text for: {}",
8549 match (g.title, g.body) {
8550 (true, true) => "title, description",
8551 (true, false) => "title",
8552 _ => "description",
8553 }
8554 )
8555 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8556 "posting the saved rewrite title and description".to_owned()
8557 } else if resolved.is_some() {
8558 "posting the owner's replacement title".to_owned()
8559 } else {
8560 "posting the neutral text for the withheld fields".to_owned()
8561 },
8562 );
8563 self.state.github_text = Some(g.clone());
8564 self.state.save()?;
8565 if g.choice == Some(GithubTextChoice::Rewrite)
8566 && let Some(r) = g.rewrite.as_ref()
8567 {
8568 return Ok(Some(PrMessage {
8569 title: r.title.clone(),
8570 body: r.body.clone(),
8571 }));
8572 }
8573 let w = gt::Withheld {
8574 title: g.title,
8575 body: g.body,
8576 categories: g.categories.clone(),
8577 };
8578 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8579 if let Some(t) = resolved {
8580 title = t;
8581 }
8582 let english = gt::LanguageDecision {
8584 title_english: true,
8585 body_english: true,
8586 body_complete: true,
8587 };
8588 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8589 if original {
8590 if g.title {
8593 title = pr.title.clone();
8594 }
8595 if g.body {
8596 body = pr.body.clone();
8597 }
8598 }
8599 Ok(Some(PrMessage { title, body }))
8600 }
8601
8602 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8603 self.state.github_text = Some(g.clone());
8604 self.state.end_pass_parked(ParkCause::Question);
8605 self.state.parked = true;
8606 self.state.event(
8607 "github-text",
8608 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8609 );
8610 self.state.save()
8611 }
8612
8613 async fn judge_pr_language(
8616 &mut self,
8617 cwd: &Path,
8618 pr: &PrMessage,
8619 ) -> Option<crate::github_text::LanguageDecision> {
8620 self.state.config.roles.language_judge.as_ref()?;
8621 if !self.state.config.graph.language_gate() {
8622 return None;
8623 }
8624 let decision =
8625 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8626 self.state.event(
8627 "github-text",
8628 if decision.is_some() {
8629 "language decided by the language judge"
8630 } else {
8631 "language judge unavailable; using built-in heuristics"
8632 },
8633 );
8634 decision
8635 }
8636
8637 async fn merge(&mut self) -> Result<()> {
8638 if self.resource_halt()? || self.hold_base_sync().await? {
8639 return Ok(());
8640 }
8641 if self
8656 .state
8657 .base_sync
8658 .as_ref()
8659 .is_some_and(|s| s.conflict.is_some())
8660 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8661 != Some(RunStatus::Gating)
8662 || !self.state.gate_allows_progress()
8670 {
8671 return Ok(());
8672 }
8673 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8682 return Ok(());
8683 }
8684 let Some(winner) = self.state.winner().cloned() else {
8685 return Ok(());
8686 };
8687 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8688 if self.state.gate_head.as_deref() != Some(head.as_str()) {
8689 self.state.status = RunStatus::Blocked;
8690 self.state.event(
8691 "gate",
8692 "winner changed after gate; fresh verification required",
8693 );
8694 self.state.save()?;
8695 return Ok(());
8696 }
8697 let wait = self.repository_wait();
8698 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8699 return Ok(());
8700 }
8701 let repo = self.state.repo.clone();
8702 let base = self.state.base_branch.clone();
8703 let mode = self.state.config.merge.mode;
8704 let style = self.state.config.merge.style;
8705 let facts = if is_review_run(&self.state) {
8706 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8707 let start = review_base(
8708 &repo,
8709 &self.state.config.merge.remote,
8710 &base,
8711 &self.state.base_commit,
8712 &winner.branch,
8713 )
8714 .await;
8715 branch_facts(&repo, &start, &winner.branch).await
8716 } else {
8717 None
8718 };
8719 let leaked = if is_review_run(&self.state) {
8722 leaked_subjects(&self.state, &winner.branch).await
8723 } else {
8724 Some(Vec::new())
8725 };
8726 let Some(pr) = self
8730 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8731 .await?
8732 else {
8733 return Ok(());
8734 };
8735 let message = pr.commit_message();
8736
8737 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
8738 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
8739 return Ok(());
8740 }
8741 let outcome = match mode {
8742 MergeMode::None => MergeOutcome {
8743 mode,
8744 ok: true,
8745 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8746 empty: false,
8747 },
8748 MergeMode::Pr | MergeMode::Local
8749 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8750 {
8751 MergeOutcome {
8752 mode,
8753 ok: false,
8754 detail: empty_candidate_detail(&self.state, &base),
8755 empty: true,
8756 }
8757 }
8758 MergeMode::Local => {
8759 let on = git::current_branch(&repo).await?;
8760 if on.as_deref() != Some(base.as_str()) {
8761 MergeOutcome {
8762 mode,
8763 ok: false,
8764 detail: format!(
8765 "{} has {} checked out, not the base branch {base}",
8766 repo.display(),
8767 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8768 ),
8769 empty: false,
8770 }
8771 } else if !git::is_clean(&repo).await? {
8772 MergeOutcome {
8773 mode,
8774 ok: false,
8775 detail: format!("{} is dirty; refusing to merge", repo.display()),
8776 empty: false,
8777 }
8778 } else {
8779 let out = match style {
8780 MergeStyle::Merge => {
8781 git::merge_no_ff(&repo, &winner.branch, &message).await?
8782 }
8783 MergeStyle::Squash => {
8784 git::merge_squash(&repo, &winner.branch, &message).await?
8785 }
8786 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8787 };
8788 MergeOutcome {
8789 mode,
8790 ok: out.ok(),
8791 detail: if out.ok() { out.stdout } else { out.stderr },
8792 empty: false,
8793 }
8794 }
8795 }
8796 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8797 self.state.pr_publication_pending = true;
8798 self.state.save()?;
8799 let out =
8800 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8801 self.state.pr_publication_pending = out.is_err();
8802 MergeOutcome {
8803 mode,
8804 ok: out.is_ok(),
8805 detail: match out {
8806 Ok(url) => url,
8807 Err(e) => format!("{e:#}"),
8808 },
8809 empty: false,
8810 }
8811 }
8812 MergeMode::Pr => {
8813 let remote = self.state.config.merge.remote.clone();
8814 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8815 let pushed = if self.state.foreign_checkout.is_some() {
8816 let refspec = format!("{delivery_head}:refs/heads/{public}");
8817 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
8818 } else {
8819 git::push(&winner.worktree, &remote, &winner.branch).await?
8820 };
8821 if !pushed.ok() {
8822 MergeOutcome {
8823 mode,
8824 ok: false,
8825 detail: pushed.stderr,
8826 empty: false,
8827 }
8828 } else {
8829 if self.state.foreign_checkout.is_some() {
8830 self.state.pr_pushed_head = Some(delivery_head.clone());
8831 }
8832 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
8838 let out = match pr_merge_plan(found) {
8839 PrPlan::Create => {
8840 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
8841 .await
8842 }
8843 PrPlan::Adopt { url, title } => {
8844 self.state
8845 .event("merge", format!("Pr: adopted open pull request {url}"));
8846 if self.state.github_text.as_ref().is_some_and(|g| {
8847 g.resolved
8848 && g.choice == Some(GithubTextChoice::Rewrite)
8849 && g.rewrite.is_some()
8850 }) {
8851 land::set_pr_text_verbatim(
8854 &winner.worktree,
8855 &url,
8856 &pr.title,
8857 &pr.body,
8858 )
8859 .await
8860 .map(|()| url)
8861 } else {
8862 if title != pr.title
8863 && (!is_review_run(&self.state)
8864 || leaked
8865 .as_deref()
8866 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8867 && let Err(e) =
8868 if self.state.github_text.as_ref().is_some_and(|g| {
8869 g.resolved
8870 && g.title
8871 && g.choice == Some(GithubTextChoice::Original)
8872 }) {
8873 land::set_pr_title_verbatim(
8875 &winner.worktree,
8876 &url,
8877 &pr.title,
8878 )
8879 .await
8880 } else {
8881 land::set_pr_title(
8882 &mut self.state,
8883 &winner.worktree,
8884 &url,
8885 &pr.title,
8886 )
8887 .await
8888 }
8889 {
8890 tracing::warn!("could not refresh title of {url}: {e:#}");
8891 self.state
8892 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8893 }
8894 Ok(url)
8895 }
8896 }
8897 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8898 };
8899 match out {
8900 Ok(url) => MergeOutcome {
8901 mode,
8902 ok: true,
8903 detail: url,
8904 empty: false,
8905 },
8906 Err(e) => MergeOutcome {
8907 mode,
8908 ok: false,
8909 detail: e.to_string(),
8910 empty: false,
8911 },
8912 }
8913 }
8914 }
8915 };
8916
8917 self.state.status = match (mode, outcome.ok) {
8918 (MergeMode::None, _) => RunStatus::Ready,
8919 (_, true) if outcome.empty => RunStatus::Ready,
8920 (MergeMode::Pr, true) => RunStatus::Ready,
8921 (_, true) => RunStatus::Merged,
8922 (_, false) => RunStatus::Blocked,
8923 };
8924 self.state.event(
8925 "merge",
8926 format!(
8927 "{:?}: {}",
8928 mode,
8929 outcome.detail.lines().next().unwrap_or("")
8930 ),
8931 );
8932 self.state.merge = Some(outcome);
8933 self.state.save()?;
8934
8935 if self.state.config.graph.land
8941 && mode == MergeMode::Pr
8942 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8943 {
8944 self.run_land().await?;
8945 }
8946 self.settle_questions();
8951 Ok(())
8952 }
8953
8954 async fn run_land(&mut self) -> Result<()> {
8965 crate::pr_head::ensure_owner(&self.state)?;
8966 let url = self
8967 .state
8968 .pr
8969 .as_ref()
8970 .map(|pr| pr.url.clone())
8971 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8972 .unwrap_or_default();
8973 let url = url.lines().next().unwrap_or("").trim().to_owned();
8974 if !url.starts_with("http") {
8975 return Ok(());
8976 }
8977 loop {
8980 match land::land(&mut self.state, &url).await {
8981 Ok(_)
8982 if self
8983 .state
8984 .merge_repairs
8985 .last()
8986 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8987 {
8988 if self.repair_merge_review().await? {
8989 continue;
8990 }
8991 }
8992 Ok(_) if self.state.resource_block.is_some() => {
8993 return self.stop_for_resource();
8994 }
8995 Ok(pr) if self.state.parked => {
8996 let _ = pr;
9000 }
9001 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9002 self.state.status = RunStatus::Superseded;
9003 self.state.save()?;
9004 }
9005 Ok(pr) => {
9006 self.state.status = match pr.state {
9007 land::PrLifecycle::Merged => RunStatus::Merged,
9008 _ => RunStatus::Blocked,
9009 };
9010 if bump::should_release_bump(self.state.status)
9017 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9018 {
9019 self.state
9025 .event("bump", format!("release bump skipped: {e:#}"));
9026 }
9027 if self.state.status == RunStatus::Merged {
9030 crate::followup::after_merge(&mut self.state, &pr.url).await;
9031 }
9032 self.state.save()?;
9033 }
9034 Err(e) => {
9035 if e.is::<crate::repo_lock::ResourceBlocked>() {
9036 return Err(e);
9037 }
9038 self.state.status = RunStatus::Blocked;
9039 self.state.event("land", format!("gave up: {e}"));
9040 self.state.save()?;
9041 }
9042 }
9043 break;
9044 }
9045 Ok(())
9046 }
9047
9048 async fn repair_merge_review(&mut self) -> Result<bool> {
9051 use crate::run::MergeRepairPhase as Phase;
9052 crate::pr_head::ensure_owner(&self.state)?;
9053 let i = self.state.merge_repairs.len() - 1;
9054 let request = self.state.merge_repairs[i].clone();
9055 if request.phase == Phase::Fixing {
9056 let restored = match self.state.winner().cloned() {
9057 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9058 None => Err(anyhow::anyhow!("repair has no winning branch")),
9059 };
9060 let why = match restored {
9061 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9062 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9063 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9064 };
9065 self.hold_merge_repair(i, why)?;
9066 return Ok(false);
9067 }
9068 if request.phase == Phase::Reviewing && self.state.gate_ran {
9073 let winner = self
9074 .state
9075 .winner()
9076 .context("repair has no winning branch")?;
9077 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9078 if self.state.gate_head.as_deref() != Some(head.as_str())
9079 || !git::is_clean(&winner.worktree).await?
9080 {
9081 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9082 return Ok(false);
9083 }
9084 }
9085 let failure = if i >= self.state.config.graph.land_rounds
9086 || self.state.config.graph.review_rounds == 0
9087 {
9088 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9089 } else {
9090 None
9091 };
9092 if let Some(why) = failure {
9093 self.hold_merge_repair(i, why)?;
9094 return Ok(false);
9095 }
9096 if request.phase == Phase::Requested {
9097 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9098 Ok(pr) => pr,
9099 Err(e) => {
9100 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9101 return Ok(false);
9102 }
9103 };
9104 self.state.merge_repairs[i].phase = Phase::Fixing;
9105 self.state.land_approval = None;
9106 self.state.land_armed_head = None;
9107 self.state.save()?;
9108 let evidence = serde_json::to_string_pretty(&request)?;
9111 let reason = format!(
9112 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9113 );
9114 let budget = self.state.config.graph.land_rounds;
9115 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9116 match result {
9117 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9118 Ok(land::Fixed::Deferred) => {
9119 let worktree = self
9123 .state
9124 .winner()
9125 .context("repair has no winning branch")?
9126 .worktree
9127 .clone();
9128 self.graft_restore(&worktree, &request.head).await?;
9129 self.state.merge_repairs[i].phase = Phase::Requested;
9130 self.stop_for_resource()?;
9131 return Ok(false);
9132 }
9133 Ok(land::Fixed::Failed(why)) => {
9134 self.hold_merge_repair(
9135 i,
9136 format!(
9137 "repair failed: {why}; inspect the branch and run an explicit review"
9138 ),
9139 )?;
9140 return Ok(false);
9141 }
9142 Err(e) => {
9143 self.hold_merge_repair(
9144 i,
9145 format!(
9146 "repair failed: {e:#}; inspect the branch and run an explicit review"
9147 ),
9148 )?;
9149 return Ok(false);
9150 }
9151 }
9152 self.begin_merge_repair_review(i)?;
9156 }
9157 self.review_loop().await?;
9158 self.gate().await?;
9159 if self.resource_halt()? || self.state.parked {
9160 return Ok(false);
9161 }
9162 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9163 != Some(RunStatus::Gating)
9164 || !self.state.gate_allows_progress()
9165 {
9166 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9167 return Ok(false);
9168 }
9169 let winner = self
9172 .state
9173 .winner()
9174 .cloned()
9175 .context("repair has no winning branch")?;
9176 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9177 if self.state.gate_head.as_deref() != Some(after.as_str())
9178 || !git::is_clean(&winner.worktree).await?
9179 {
9180 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9181 return Ok(false);
9182 }
9183 let expected = self
9184 .state
9185 .pr_pushed_head
9186 .as_deref()
9187 .unwrap_or(&request.head)
9188 .to_owned();
9189 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9190 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9193 return Ok(false);
9194 }
9195 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9196 let refspec = format!("{after}:refs/heads/{public}");
9197 let push = git::git_raw(
9198 &winner.worktree,
9199 &["push", &lease, &self.state.config.merge.remote, &refspec],
9200 )
9201 .await;
9202 match push {
9203 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9204 Ok(out) => {
9205 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9206 return Ok(false);
9207 }
9208 Err(e) => {
9209 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9210 return Ok(false);
9211 }
9212 }
9213 self.state.merge_repairs[i].phase = Phase::Complete;
9214 self.state.status = RunStatus::Landing;
9215 self.state.event(
9216 "repair",
9217 "new review and gate completed; fresh owner merge decision required",
9218 );
9219 self.state.save()?;
9220 Ok(true)
9221 }
9222
9223 fn merge_repair_artifacts(&self) -> PathBuf {
9224 let artifacts = agent::artifacts_dir(&self.state.dir());
9225 match self
9226 .state
9227 .merge_repairs
9228 .last()
9229 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9230 {
9231 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9232 None => artifacts,
9233 }
9234 }
9235
9236 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9237 if let Some(e2e) = self.state.head_e2e.take() {
9238 self.state
9239 .verification_history
9240 .push(crate::run::VerificationEvidence {
9241 superseded_by: None,
9242 archived_at: Some(Timestamp::now()),
9243 fixes_spent: Some(self.state.gate_fixes.len()),
9244 gate_head: None,
9245 gate_ran: false,
9246 gate: Vec::new(),
9247 e2e: Some(e2e),
9248 });
9249 }
9250 self.state.reviews.clear();
9251 self.state.gate.clear();
9252 self.state.gate_ran = false;
9253 self.state.gate_head = None;
9254 self.state.gate_fixes.clear();
9255 self.state.gate_repair_e2e = None;
9256 self.state.pre_gate.clear();
9257 self.state.pre_gate_commit = None;
9258 self.state.contested_handoff = None;
9259 self.state.land_reruns = None;
9260 self.state.land_approval = None;
9261 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9262 self.state.status = RunStatus::Reviewing;
9263 self.state.save()
9264 }
9265
9266 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9267 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9268 self.state.merge_repairs[index].failure = Some(why.clone());
9269 self.state.status = RunStatus::Blocked;
9270 self.state.event("repair", why);
9271 self.state.save()
9272 }
9273
9274 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9278 if let Some(existing) = self.state.seats.get(key)
9279 && existing.agent == agent
9280 {
9281 return existing.clone();
9282 }
9283 let fresh = if self.state.seats.contains_key(key) {
9286 handover_seat(key, agent, self.state.next_seat_seed())
9287 } else {
9288 SeatState::new(key, agent, self.state.seed)
9289 };
9290 self.state.seats.insert(key.to_owned(), fresh.clone());
9291 fresh
9292 }
9293
9294 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9300 match self.state.seats.get(key) {
9301 Some(s) if s.agent != spec.id => {
9302 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9303 }
9304 _ => spec,
9305 }
9306 }
9307
9308 fn view(&self, c: &Candidate) -> CandidateView {
9310 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9311 .unwrap_or_default();
9312 let (patch, _) = blind::sanitize_patch(
9313 &format!("candidate {} patch", c.label),
9314 &raw,
9315 &self.state.config.blind,
9316 );
9317 CandidateView {
9318 label: c.label,
9319 branch: c.branch.clone(),
9320 summary: c.summary.clone(),
9321 stat: c.stat.clone(),
9322 patch,
9323 }
9324 }
9325
9326 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9328 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9329 prompt::judge(
9330 "(see above)",
9331 &views,
9332 self.roles.judges.len(),
9333 base_short,
9334 "en",
9335 )
9336 }
9337
9338 fn vote_prompt_full(
9344 &self,
9345 j: usize,
9346 labels: &[char],
9347 language: &str,
9348 candidates: &[Candidate],
9349 base_short: &str,
9350 ) -> String {
9351 let mut text = format!(
9352 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9353 prompt::final_vote(labels, language),
9354 self.state.instruction,
9355 self.candidate_block(candidates, base_short)
9356 );
9357 if let Some(own) = self
9358 .state
9359 .judgements
9360 .get(j)
9361 .filter(|r| !r.ranking.is_empty())
9362 {
9363 let reasons = own
9364 .reasons
9365 .iter()
9366 .map(|(k, v)| format!("- {k}: {v}"))
9367 .collect::<Vec<_>>()
9368 .join("\n");
9369 text.push_str(&format!(
9370 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9371 own.ranking.iter().collect::<String>(),
9372 if reasons.is_empty() {
9373 ""
9374 } else {
9375 ", because:\n"
9376 }
9377 ));
9378 }
9379 if !self.state.deliberation.is_empty() {
9380 text.push_str("\n# What was argued before this vote\n");
9381 for t in self.transcript(&[], j) {
9382 text.push_str(&format!(
9383 "\n## {}{}\n\n{}\n",
9384 t.who,
9385 if t.is_self { " (you)" } else { "" },
9386 t.body.trim()
9387 ));
9388 }
9389 }
9390 text
9391 }
9392
9393 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9400 let mut turns = Vec::new();
9401 for j in &self.state.judgements {
9402 if j.ranking.is_empty() {
9403 continue;
9404 }
9405 let reasons = j
9406 .reasons
9407 .iter()
9408 .map(|(k, v)| format!("- {k}: {v}"))
9409 .collect::<Vec<_>>()
9410 .join("\n");
9411 turns.push(Turn {
9412 who: format!("Judge {} (opening ranking)", j.judge),
9413 is_self: j.judge == self_idx + 1,
9414 body: format!(
9415 "Ranked {}{}{reasons}",
9416 j.ranking.iter().collect::<String>(),
9417 if reasons.is_empty() {
9418 ""
9419 } else {
9420 ", because:\n"
9421 }
9422 ),
9423 });
9424 }
9425 for t in self
9426 .state
9427 .deliberation
9428 .iter()
9429 .flat_map(|r| r.turns.iter())
9430 .chain(current)
9431 {
9432 turns.push(Turn {
9433 who: format!("Judge {}", t.judge),
9434 is_self: t.judge == self_idx + 1,
9435 body: t.body.clone(),
9436 });
9437 }
9438 turns
9439 }
9440}
9441
9442fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9444 agent::has_session(spec.kind, seat, sessions)
9445}
9446
9447fn next_untried_in_roster<'a>(
9468 roster: &'a [AgentSpec],
9469 start: usize,
9470 tried: &BTreeSet<String>,
9471) -> Option<&'a AgentSpec> {
9472 roster
9473 .get(start + 1..)?
9474 .iter()
9475 .find(|s| !tried.contains(&s.id))
9476}
9477
9478fn pick_successor<'a>(
9499 roster: &'a [AgentSpec],
9500 start: usize,
9501 tried: &BTreeSet<String>,
9502 carried: Option<&BTreeSet<String>>,
9503 others: &BTreeSet<String>,
9504) -> Option<&'a AgentSpec> {
9505 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9506 roster
9507 .get(start + 1..)
9508 .and_then(|tail| {
9509 tail.iter()
9510 .filter(free)
9511 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9512 })
9513 .or_else(|| {
9514 carried
9515 .is_some()
9516 .then(|| roster.iter().find(free))
9517 .flatten()
9518 })
9519 .or_else(|| match carried {
9520 Some(c) => next_for_seat(roster, start, tried, c),
9521 None => next_untried_in_roster(roster, start, tried),
9522 })
9523}
9524
9525fn next_for_seat<'a>(
9535 roster: &'a [AgentSpec],
9536 start: usize,
9537 round_tried: &BTreeSet<String>,
9538 carried_failed: &BTreeSet<String>,
9539) -> Option<&'a AgentSpec> {
9540 roster
9541 .get(start + 1..)?
9542 .iter()
9543 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9544 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9545}
9546
9547fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9553 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9554 return spec;
9555 };
9556 let ok = |id: &str| !h.failed.contains(id);
9557 if let Some(last) = h.last_ok.as_deref()
9558 && ok(last)
9559 && let Some(s) = roster.iter().find(|s| s.id == last)
9560 {
9561 return s.clone();
9562 }
9563 if ok(&spec.id) {
9564 return spec;
9565 }
9566 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9567 roster
9568 .get(from + 1..)
9569 .into_iter()
9570 .flatten()
9571 .chain(roster.iter())
9572 .find(|s| ok(&s.id))
9573 .cloned()
9574 .unwrap_or(spec)
9575}
9576
9577pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9581 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9582}
9583
9584const TIMED_OUT: &str = "timed out";
9587
9588impl FailClass {
9589 fn of(out: &AgentOutcome) -> Option<Self> {
9591 match out {
9592 AgentOutcome::Ok(_) => None,
9593 AgentOutcome::Quota(_) => Some(Self::Quota),
9594 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9595 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9596 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9597 AgentOutcome::ResourceBlocked(_) => None,
9600 }
9601 }
9602
9603 fn stem_word(&self) -> &'static str {
9605 match self {
9606 Self::Quota => "quota",
9607 _ => "handover",
9608 }
9609 }
9610}
9611
9612fn failure_signature(msg: &str) -> String {
9616 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9617 if let Some(code) = line
9620 .strip_prefix("process exited with no usable output (exit code ")
9621 .and_then(|s| s.strip_suffix(')'))
9622 .and_then(|s| s.parse::<i32>().ok())
9623 {
9624 line = format!("exited with some({code}) and no usable output");
9625 } else if matches!(
9626 line.as_str(),
9627 "process exited with no usable output (no exit code reported)"
9628 | "process exited with no usable output (exit code unavailable)"
9629 ) {
9630 line = "exited with none and no usable output".to_owned();
9631 }
9632 let mut out = Vec::new();
9633 for word in line.split_whitespace() {
9634 if word.contains('/') || word.contains('\\') {
9635 out.push("<path>".to_owned());
9636 continue;
9637 }
9638 let mut w = String::new();
9639 let mut in_digits = false;
9640 for c in word.chars() {
9641 if c.is_ascii_digit() {
9642 if !in_digits {
9643 w.push('#');
9644 }
9645 in_digits = true;
9646 } else {
9647 in_digits = false;
9648 w.push(c);
9649 }
9650 }
9651 out.push(w);
9652 }
9653 out.join(" ").chars().take(120).collect()
9654}
9655
9656fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9663 match cur {
9664 FailClass::Quota | FailClass::Timeout => true,
9665 FailClass::Other(_) => prev != Some(cur),
9666 }
9667}
9668
9669fn fail_reason(out: &AgentOutcome) -> String {
9671 match out {
9672 AgentOutcome::Ok(_) => String::new(),
9673 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9674 AgentOutcome::Dropped(o) => format!(
9675 "the CLI dropped the stream ({})",
9676 o.dropped
9677 .as_ref()
9678 .map(|d| d.why.as_str())
9679 .unwrap_or("it ended without delivering its answer")
9680 ),
9681 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9682 e.lines().next().unwrap_or("").chars().take(160).collect()
9683 }
9684 }
9685}
9686
9687pub(crate) fn record_handover(
9690 state: &mut RunState,
9691 node: &str,
9692 seat: &str,
9693 from: &str,
9694 to: &str,
9695 class: &FailClass,
9696 reason: &str,
9697) {
9698 let message = if *class == FailClass::Quota {
9699 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9700 } else {
9701 format!("{seat}: handed over {from} -> {to} ({reason})")
9702 };
9703 state.event(node, message);
9704 state.handovers.push(Handover {
9705 at: Timestamp::now(),
9706 node: node.to_owned(),
9707 seat: seat.to_owned(),
9708 from: from.to_owned(),
9709 to: to.to_owned(),
9710 reason: reason.to_owned(),
9711 });
9712}
9713
9714fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9728 commands.iter().any(|c| c.exit_code.is_none())
9729}
9730
9731fn verified_noop_claim(
9744 usable: bool,
9745 commands: &[agent::CommandEvidence],
9746 text: &str,
9747) -> Option<String> {
9748 (usable && !has_unconfirmed_command(commands))
9749 .then(|| verdict::verified_noop(text))
9750 .flatten()
9751}
9752
9753fn short(commit: &str) -> String {
9754 commit.chars().take(7).collect()
9755}
9756
9757fn make_executable(path: &Path) -> Result<()> {
9758 #[cfg(unix)]
9759 {
9760 use std::os::unix::fs::PermissionsExt as _;
9761 let mut perms = std::fs::metadata(path)?.permissions();
9762 perms.set_mode(0o755);
9763 std::fs::set_permissions(path, perms)?;
9764 }
9765 #[cfg(not(unix))]
9766 {
9767 let _ = path;
9768 }
9769 Ok(())
9770}
9771
9772struct WaveCtx<'a> {
9779 run: &'a str,
9782 node: &'a str,
9784 prompts: &'a Prompts,
9785 cache: Option<&'a Path>,
9787 round: Option<usize>,
9790 carry_seats: bool,
9794 pauses: Option<(&'a Pause, &'a Pause)>,
9797}
9798
9799fn repository_wait(
9801 state: &RunState,
9802 pauses: Option<(&Pause, &Pause)>,
9803) -> crate::repo_lock::WaitPolicy {
9804 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9805 let inherited = crate::repo_lock::WaitPolicy::current();
9806 crate::repo_lock::WaitPolicy::new(
9807 Duration::from_secs(state.config.graph.timeout_resource_wait),
9808 move || {
9809 inherited.cancelled()
9810 || pauses
9811 .as_ref()
9812 .is_some_and(|(a, b)| a.parked() || b.parked())
9813 },
9814 )
9815}
9816
9817fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9818 if error.is::<crate::repo_lock::ResourceBlocked>() {
9819 state.resource_block = Some(crate::run::ResourceBlock {
9820 cause: crate::run::ResourceBlockCause::Other,
9821 node: "repository".into(),
9822 kind: if error.to_string().contains("disk pressure") {
9823 "disk pressure"
9824 } else {
9825 "repository coordination"
9826 }
9827 .into(),
9828 detail: error.to_string(),
9829 at: Timestamp::now(),
9830 });
9831 state.event("resource", error.to_string());
9832 }
9833}
9834
9835async fn coordinate_operation(
9836 state: &mut RunState,
9837 cwd: &Path,
9838 wait: &crate::repo_lock::WaitPolicy,
9839) -> Result<Option<crate::repo_lock::SeatLease>> {
9840 let result =
9841 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9842 .await;
9843 if let Err(e) = &result {
9844 record_repository_block(state, e);
9845 }
9846 result
9847}
9848
9849async fn run_one(
9851 job: SeatJob,
9852 sem: Arc<Semaphore>,
9853 ctx: &WaveCtx<'_>,
9854 state: &mut RunState,
9855 attempt: usize,
9856) -> (SeatState, AgentOutcome) {
9857 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9858 .await
9859 .pop()
9860 .expect("one job in, one result out");
9861 (seat, out)
9862}
9863
9864async fn wave(
9870 jobs: Vec<SeatJob>,
9871 sem: Arc<Semaphore>,
9872 ctx: &WaveCtx<'_>,
9873 state: &mut RunState,
9874 attempt: usize,
9875) -> Vec<(usize, SeatState, AgentOutcome)> {
9876 let WaveCtx {
9877 run,
9878 node,
9879 prompts,
9880 cache,
9881 round,
9882 carry_seats: _,
9883 pauses,
9884 } = *ctx;
9885 for job in &jobs {
9886 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9887 }
9888 if let Err(e) = state.save() {
9889 tracing::warn!("could not persist in-progress seats: {e:#}");
9894 }
9895 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9911 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9915 let cache_guard = if let Some(cache_dir) = cache {
9916 if jobs_had_a_writer {
9917 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9918 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9919 match acquire_cache_lease(
9920 state,
9921 cache_dir,
9922 &owner,
9923 resource_limit,
9924 node,
9925 node,
9926 &cancel,
9927 )
9928 .await
9929 {
9930 Ok(Some(g)) => Some(g),
9931 Ok(None) => {
9934 if let Some(block) = &mut state.resource_block {
9935 block.cause = crate::run::ResourceBlockCause::Cancelled {
9936 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9937 };
9938 }
9939 return block_wave(
9940 jobs,
9941 state,
9942 node,
9943 attempt,
9944 "parked while waiting for the shared build cache",
9945 );
9946 }
9947 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9950 Err(e) => {
9951 let why = format!("{e:#}");
9955 state.event(
9956 "resource",
9957 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9958 );
9959 return block_wave(jobs, state, node, attempt, &why);
9960 }
9961 }
9962 } else {
9963 None
9964 }
9965 } else {
9966 None
9967 };
9968 state.seats_spawning(node, attempt);
9971 if let Err(e) = state.save() {
9972 tracing::warn!("could not persist the start of a wave: {e:#}");
9973 }
9974 let slot_limit = resource_limit
9978 + jobs
9979 .iter()
9980 .map(|j| j.timeout)
9981 .max()
9982 .unwrap_or_default()
9983 .saturating_mul(jobs.len() as u32);
9984 let mut set = tokio::task::JoinSet::new();
9985 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9988 let overlay = prompts.overlay(node);
9989 let repository_wait = repository_wait(state, pauses);
9990 for (i, mut job) in jobs.into_iter().enumerate() {
9991 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9992 if cache.is_some() {
9993 job.prompt.push('\n');
9994 job.prompt
9995 .push_str(&prompt::build_cache_note(node, job.allow_write));
9996 }
9997 let sem = Arc::clone(&sem);
9998 let run = run.to_owned();
9999 let node = node.to_owned();
10000 let attachments = if node == "implement" {
10003 state.attachments.clone()
10004 } else {
10005 Vec::new()
10006 };
10007 let cache = cache
10018 .filter(|_| job.allow_write && cache_guard.is_some())
10019 .map(Path::to_path_buf);
10020 let spawned_tx = spawned_tx.clone();
10021 let repository_wait = repository_wait.clone();
10022 let coordination = state.config.coordination.clone();
10023 let fetched_base = state.fetched_base.clone();
10024 set.spawn(async move {
10025 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10029 Ok(p) => p,
10030 Err(_) => {
10031 return (
10032 i,
10033 job.seat,
10034 AgentOutcome::ResourceBlocked(format!(
10035 "no parallel slot became free within {}s",
10036 slot_limit.as_secs()
10037 )),
10038 false,
10039 );
10040 }
10041 };
10042 spawned_tx.send(job.seat.key.clone()).ok();
10043 let mut seat = job.seat;
10044 let out = repository_wait
10045 .scope(agent::invoke(
10046 &job.spec,
10047 &mut seat,
10048 &Invocation {
10049 coordination: coordination.as_ref(),
10050 fetched_base: fetched_base.as_ref(),
10051 cwd: &job.cwd,
10052 prompt: &job.prompt,
10053 timeout: job.timeout,
10054 allow_write: job.allow_write,
10055 sessions: job.sessions,
10056 artifacts: &job.artifacts,
10057 stem: &job.stem,
10058 run: &run,
10059 node: &node,
10060 cache_dir: cache.as_deref(),
10061 attachments: &attachments,
10062 writable: &[],
10063 },
10064 ))
10065 .await;
10066 let interrupted = out
10067 .as_ref()
10068 .err()
10069 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10070 let out = match out {
10071 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10072 Ok(o) if o.usable() => AgentOutcome::Ok(o),
10073 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10081 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10082 Ok(o) => AgentOutcome::Failed(format!(
10083 "process exited with no usable output ({})",
10084 crate::notices::exit_status(o.exit_code)
10085 )),
10086 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10087 AgentOutcome::ResourceBlocked(e.to_string())
10088 }
10089 Err(e) => AgentOutcome::Failed(e.to_string()),
10090 };
10091 (i, seat, out, interrupted)
10092 });
10093 }
10094 drop(spawned_tx);
10095 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10096 loop {
10097 let joined = tokio::select! {
10098 Some(key) = spawned_rx.recv() => {
10099 state.seat_spawned(&key);
10101 if let Err(e) = state.save() {
10102 tracing::warn!("could not persist a seat's start: {e:#}");
10103 }
10104 continue;
10105 }
10106 joined = set.join_next() => joined,
10107 };
10108 let Some(joined) = joined else {
10109 break;
10110 };
10111 let (i, seat, out, interrupted) = match joined {
10112 Ok(v) => v,
10113 Err(e) => {
10117 tracing::error!("agent task panicked: {e}");
10118 continue;
10119 }
10120 };
10121 state.seat_finished(&seat.key);
10122 if interrupted
10123 && node == "implement"
10124 && let Some(label) = state
10125 .candidates
10126 .iter()
10127 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10128 .map(|candidate| candidate.label)
10129 {
10130 state.candidate_preservation.insert(
10131 label,
10132 "interrupted implementation awaiting persistence".into(),
10133 );
10134 state.interrupted_implementations.insert(label);
10135 }
10136 if let AgentOutcome::ResourceBlocked(why) = &out
10137 && state.resource_block.is_none()
10138 {
10139 state.resource_block = Some(crate::run::ResourceBlock {
10140 cause: crate::run::ResourceBlockCause::Other,
10141 node: node.to_owned(),
10142 kind: if why.contains("disk pressure") {
10143 "disk pressure"
10144 } else if why.contains("repository coordination") {
10145 "repository coordination"
10146 } else {
10147 "parallel slot"
10148 }
10149 .to_owned(),
10150 detail: why.clone(),
10151 at: Timestamp::now(),
10152 });
10153 let activity = if interrupted {
10154 "agent execution interrupted"
10155 } else {
10156 "no agent was started"
10157 };
10158 state.event("resource", format!("{node}: {activity} - {why}"));
10159 }
10160 record_jobs(state, node, round, &seat.key, &out);
10161 if let Err(e) = state.save() {
10162 tracing::warn!("could not persist a seat's completion: {e:#}");
10163 }
10164 if collected.len() <= i {
10165 collected.resize_with(i + 1, || None);
10166 }
10167 collected[i] = Some((i, seat, out));
10168 }
10169 if state
10175 .active
10176 .values()
10177 .any(|a| a.node == node && a.attempt == attempt)
10178 {
10179 state
10180 .active
10181 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10182 if let Err(e) = state.save() {
10183 tracing::warn!("could not persist the end of a wave: {e:#}");
10184 }
10185 }
10186 if let Some(cache_dir) = cache
10193 && jobs_had_a_writer
10194 {
10195 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10196 }
10197 if let Some(guard) = cache_guard {
10198 guard.release();
10199 }
10200 collected.into_iter().flatten().collect()
10201}
10202
10203fn record_jobs(
10214 state: &mut RunState,
10215 node: &str,
10216 round: Option<usize>,
10217 seat: &str,
10218 out: &AgentOutcome,
10219) {
10220 let commands: &[agent::CommandEvidence] = match out {
10221 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10222 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10223 };
10224 let checked_at = Timestamp::now();
10225 for c in commands {
10226 state.jobs.push(JobRecord {
10227 node: node.to_owned(),
10228 round,
10229 seat: seat.to_owned(),
10230 id: c.id.clone(),
10231 description: c.description.clone(),
10232 checked_at,
10233 status: match c.exit_code {
10234 Some(0) => JobStatus::Completed,
10235 Some(_) => JobStatus::Failed,
10236 None => JobStatus::Unknown,
10237 },
10238 exit_code: c.exit_code,
10239 result_summary: c.result_summary.clone(),
10240 source: c.source.clone(),
10241 });
10242 }
10243}
10244
10245fn round_is_clean(
10266 blocking: usize,
10267 e2e_ok: bool,
10268 answered: usize,
10269 expected: usize,
10270 quota_missing: usize,
10271 policy: IncompleteReviewPolicy,
10272) -> bool {
10273 if blocking != 0 || !e2e_ok {
10274 return false;
10275 }
10276 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10277 return true;
10278 }
10279 answered > 0 && expected - answered <= quota_missing
10280}
10281
10282pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10306 if max_rounds == 0
10307 || reviews.last().is_some_and(|r| {
10308 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10309 })
10310 {
10311 return Some(RunStatus::Gating);
10312 }
10313 let last = reviews.last()?;
10314 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10315 if reviews.len() < max_rounds && !stagnant {
10316 return None;
10317 }
10318 if last.incomplete() && last.blocking == 0 {
10319 return Some(RunStatus::Blocked);
10320 }
10321 if last.e2e_status() == E2eStatus::ResourceBlocked
10322 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10323 {
10324 return None;
10325 }
10326 Some(if last.e2e.iter().all(CommandOutcome::ok) {
10327 RunStatus::Gating
10328 } else {
10329 RunStatus::Blocked
10330 })
10331}
10332
10333pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10348 if nudged {
10349 (full / 4).max(Duration::from_secs(120)).min(full)
10350 } else {
10351 full
10352 }
10353}
10354
10355#[allow(clippy::too_many_arguments)]
10384async fn ask_json_wave<T>(
10385 jobs: Vec<SeatJob>,
10386 sem: Arc<Semaphore>,
10387 retries: usize,
10388 roster: &[AgentSpec],
10389 ctx: &WaveCtx<'_>,
10390 losses: &mut Vec<QuotaLoss>,
10391 state: &mut RunState,
10392 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10393) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10394where
10395 T: serde::de::DeserializeOwned + Send + 'static,
10396{
10397 ask_wave_with(
10398 jobs,
10399 sem,
10400 retries,
10401 roster,
10402 ctx,
10403 losses,
10404 state,
10405 &|text: &str| {
10406 let v = verdict::extract_json::<T>(text)?;
10407 validate(&v)?;
10408 Ok(v)
10409 },
10410 )
10411 .await
10412}
10413
10414#[allow(clippy::too_many_arguments)]
10423async fn ask_wave_with<T>(
10424 jobs: Vec<SeatJob>,
10425 sem: Arc<Semaphore>,
10426 retries: usize,
10427 roster: &[AgentSpec],
10428 ctx: &WaveCtx<'_>,
10429 losses: &mut Vec<QuotaLoss>,
10430 state: &mut RunState,
10431 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10432) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10433where
10434 T: Send + 'static,
10435{
10436 let n = jobs.len();
10437 let originals: Vec<SeatJob> = jobs;
10438 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10439 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10440 let mut nudges: Vec<usize> = vec![0; n];
10447 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10451 let mut tried: Vec<BTreeSet<String>> = specs
10452 .iter()
10453 .map(|s| BTreeSet::from([s.id.clone()]))
10454 .collect();
10455 let starts: Vec<usize> = specs
10456 .iter()
10457 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10458 .collect();
10459 let carry = ctx.carry_seats && !roster.is_empty();
10460 let carried: Vec<BTreeSet<String>> = originals
10463 .iter()
10464 .map(|j| {
10465 state
10466 .seat_history
10467 .get(&j.seat.key)
10468 .filter(|_| carry)
10469 .map(|h| h.failed.clone())
10470 .unwrap_or_default()
10471 })
10472 .collect();
10473 let mut prev: Vec<Option<FailClass>> = originals
10474 .iter()
10475 .map(|j| {
10476 state
10477 .seat_history
10478 .get(&j.seat.key)
10479 .filter(|_| carry)
10480 .and_then(|h| h.last_fail.clone())
10481 })
10482 .collect();
10483 let next_agent =
10484 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10485 let others: BTreeSet<String> = specs
10486 .iter()
10487 .enumerate()
10488 .filter(|(j, _)| *j != i)
10489 .map(|(_, s)| s.id.clone())
10490 .collect();
10491 pick_successor(
10492 roster,
10493 starts[i],
10494 tried,
10495 carry.then(|| &carried[i]),
10496 &others,
10497 )
10498 .cloned()
10499 };
10500 let mut fresh: Vec<Option<String>> = vec![None; n];
10501 let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
10502 let mut pending: Vec<usize> = (0..n).collect();
10503
10504 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10509 for round in 0..max_rounds {
10510 if pending.is_empty() {
10511 break;
10512 }
10513 let mut batch = Vec::with_capacity(pending.len());
10514 let mut renudged: Vec<&str> = Vec::new();
10515 for &i in &pending {
10516 let src = &originals[i];
10517 let full: &str = match &src.handover {
10522 Some(h) if specs[i].id != src.spec.id => h,
10523 _ => &src.prompt,
10524 };
10525 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10528 (
10529 full.to_owned(),
10530 src.timeout,
10531 format!("{}-{word}-{}", src.stem, specs[i].id),
10532 )
10533 } else if nudges[i] == 0 {
10534 (full.to_owned(), src.timeout, src.stem.clone())
10535 } else {
10536 renudged.push(src.seat.key.as_str());
10537 let why = done[i]
10538 .as_ref()
10539 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10540 .unwrap_or_else(|| "no parsable answer".to_owned());
10541 let nudge = prompt::nudge(&why);
10542 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10543 let prompt = if nudged {
10544 nudge
10545 } else {
10546 format!("{full}\n\n---\n\n{nudge}")
10547 };
10548 (
10549 prompt,
10550 retry_budget(src.timeout, nudged),
10551 format!("{}-retry{}", src.stem, nudges[i]),
10552 )
10553 };
10554 batch.push(SeatJob {
10555 spec: specs[i].clone(),
10556 seat: seats[i].clone(),
10557 cwd: src.cwd.clone(),
10558 prompt,
10559 timeout,
10560 allow_write: src.allow_write,
10561 sessions: src.sessions,
10562 artifacts: src.artifacts.clone(),
10563 stem,
10564 handover: None,
10565 });
10566 }
10567
10568 if !renudged.is_empty() {
10569 state.event(
10570 ctx.node,
10571 format!("retry {round}: re-asking {}", renudged.join(", ")),
10572 );
10573 }
10574 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10575 let mut still = Vec::new();
10576 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10577 seats[i] = seat;
10578 if let AgentOutcome::ResourceBlocked(why) = &out {
10581 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10582 continue;
10583 }
10584 let class = FailClass::of(&out);
10585 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10588 && nudges[i] < retries
10589 && !roster.is_empty();
10590 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10591 let next = should_hand_over(prev[i].as_ref(), &cur)
10592 .then(|| next_agent(i, &tried[i], &specs))
10593 .flatten();
10594 if carry {
10595 let h = state
10596 .seat_history
10597 .entry(originals[i].seat.key.clone())
10598 .or_default();
10599 h.failed.insert(specs[i].id.clone());
10600 h.last_fail = Some(cur.clone());
10601 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10602 h.last_ok = None;
10603 }
10604 if let Err(e) = state.save() {
10607 tracing::warn!("could not persist a seat's failure history: {e:#}");
10608 }
10609 }
10610 if let Some(next) = next {
10611 record_handover(
10612 state,
10613 ctx.node,
10614 &originals[i].seat.key,
10615 &specs[i].id,
10616 &next.id,
10617 &cur,
10618 &fail_reason(&out),
10619 );
10620 tried[i].insert(next.id.clone());
10621 prev[i] = Some(cur.clone());
10622 seats[i] =
10623 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10624 specs[i] = next;
10625 fresh[i] = Some(cur.stem_word().to_owned());
10626 nudges[i] = 0;
10627 done[i] = Some(Err(anyhow::anyhow!(
10628 "handed over after: {}",
10629 fail_reason(&out)
10630 )));
10631 still.push(i);
10632 continue;
10633 }
10634 }
10635 if carry
10636 && !nudge_first
10637 && let Some(cur) = class.clone()
10638 {
10639 let h = state
10641 .seat_history
10642 .entry(originals[i].seat.key.clone())
10643 .or_default();
10644 h.failed.insert(specs[i].id.clone());
10645 h.last_fail = Some(cur);
10646 }
10647 let parsed = match out {
10648 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10649 AgentOutcome::Quota(o) => {
10650 last_quota[i] = o.quota.clone();
10651 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10652 }
10653 AgentOutcome::Dropped(o) => {
10657 let why = o
10658 .dropped
10659 .as_ref()
10660 .map(|d| d.why.as_str())
10661 .unwrap_or("the CLI ended the stream without delivering its answer");
10662 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10663 }
10664 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10665 Err(anyhow::anyhow!(e))
10666 }
10667 };
10668 let quota = class == Some(FailClass::Quota);
10669 let failed = parsed.is_err();
10670 if !failed && matches!(ctx.node, "judge" | "vote") {
10671 state
10674 .quota
10675 .retain(|loss| loss.seat != originals[i].seat.key);
10676 }
10677 done[i] = Some(parsed);
10678 if carry && !failed {
10679 let h = state
10680 .seat_history
10681 .entry(originals[i].seat.key.clone())
10682 .or_default();
10683 h.failed.remove(&specs[i].id);
10684 h.last_ok = Some(specs[i].id.clone());
10685 h.last_fail = None;
10686 }
10687 let agent_failure = class.is_some()
10695 && !nudge_first
10696 && !roster.is_empty()
10697 && next_agent(i, &tried[i], &specs).is_some();
10698 if failed && !quota && !agent_failure && nudges[i] < retries {
10699 nudges[i] += 1;
10700 still.push(i);
10701 }
10702 }
10703 pending = still;
10704 }
10705
10706 for (i, q) in last_quota.into_iter().enumerate() {
10709 if let Some(quota) = q {
10710 losses.push(QuotaLoss::from_report(
10711 originals[i].seat.key.clone(),
10712 ctx.node,
10713 Some("a),
10714 ));
10715 }
10716 }
10717
10718 if state.status != RunStatus::Stalled
10721 && ctx.node != "deliberate"
10722 && n > 0
10723 && losses.len() == n
10724 && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
10725 {
10726 block_machine_outage(state, ctx.node, losses);
10727 state.quota.append(losses);
10728 }
10729
10730 seats
10731 .into_iter()
10732 .zip(done)
10733 .zip(nudges)
10734 .map(|((seat, res), attempts)| {
10735 (
10736 seat,
10737 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10738 attempts,
10739 )
10740 })
10741 .collect()
10742}
10743
10744pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
10747 let mut auth: Vec<_> = losses
10748 .iter()
10749 .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
10750 .map(|loss| loss.agent.clone())
10751 .collect();
10752 auth.sort();
10753 auth.dedup();
10754 let (kind, detail) = if auth.is_empty() {
10755 (
10756 "agent usage limit",
10757 "All seats reached their agent usage limits; resume after usage is available"
10758 .to_owned(),
10759 )
10760 } else {
10761 (
10762 "agent authentication",
10763 format!(
10764 "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
10765 auth.join(", ")
10766 ),
10767 )
10768 };
10769 state.resource_block = Some(crate::run::ResourceBlock {
10770 node: node.to_owned(),
10771 kind: kind.to_owned(),
10772 cause: crate::run::ResourceBlockCause::Other,
10773 detail: detail.clone(),
10774 at: Timestamp::now(),
10775 });
10776 state.event(node, detail);
10777}
10778
10779fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10781 results
10782 .iter()
10783 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10784}
10785
10786fn block_wave(
10790 jobs: Vec<SeatJob>,
10791 state: &mut RunState,
10792 node: &str,
10793 attempt: usize,
10794 why: &str,
10795) -> Vec<(usize, SeatState, AgentOutcome)> {
10796 state.resource_wait = None;
10797 state
10798 .active
10799 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10800 if let Err(e) = state.save() {
10801 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10802 }
10803 jobs.into_iter()
10804 .enumerate()
10805 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10806 .collect()
10807}
10808
10809async fn acquire_cache_lease(
10822 state: &mut RunState,
10823 cache_dir: &Path,
10824 owner: &crate::cache::Owner,
10825 budget: Duration,
10826 node: &str,
10827 context: &str,
10828 cancel: &(dyn Fn() -> bool + Sync),
10829) -> Result<Option<crate::cache::Guard>> {
10830 let home = crate::run::home();
10831 let started = Instant::now();
10832 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10833 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10834 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10835 Err(e) => {
10836 state.event(
10837 "verify",
10838 format!("{context}: could not check the shared build cache: {e:#}"),
10839 );
10840 if let Err(e2) = state.save() {
10841 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10842 }
10843 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10844 }
10845 };
10846 state.event(
10847 "verify",
10848 format!(
10849 "{context}: waiting for the shared build cache at {} ({})",
10850 cache_dir.display(),
10851 busy.describe()
10852 ),
10853 );
10854 state.resource_wait = Some(crate::run::ResourceWait {
10857 node: context.to_owned(),
10858 kind: "shared build cache".to_owned(),
10859 holder: busy.describe(),
10860 since: Timestamp::now(),
10861 limit_secs: budget.as_secs(),
10862 });
10863 if let Err(e) = state.save() {
10864 tracing::warn!("could not persist a cache wait: {e:#}");
10865 }
10866 let remaining = budget.saturating_sub(started.elapsed());
10867 let waited = crate::cache::wait_for_unless(
10868 &home,
10869 cache_dir,
10870 owner,
10871 remaining,
10872 Duration::from_secs(5),
10873 cancel,
10874 )
10875 .await;
10876 state.resource_wait = None;
10877 let cause = match &waited {
10878 Ok(Some(_)) => None,
10879 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10880 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10881 Some(crate::run::ResourceBlockCause::Timeout {
10882 waited_secs: started.elapsed().as_secs(),
10883 limit_secs: budget.as_secs(),
10884 })
10885 }
10886 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10887 };
10888 if let Some(cause) = cause {
10889 let detail = match &waited {
10890 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10891 Err(error) => format!("{error:#}"),
10892 };
10893 state.resource_block = Some(crate::run::ResourceBlock {
10894 cause,
10895 node: node.to_owned(),
10896 kind: "shared build cache".into(),
10897 detail,
10898 at: Timestamp::now(),
10899 });
10900 state.event(
10901 "resource",
10902 state.resource_block.as_ref().unwrap().description(),
10903 );
10904 }
10905 if let Err(error) = state.save() {
10906 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10907 }
10908 waited
10909}
10910
10911#[derive(Debug)]
10916struct CacheCheckError(String);
10917
10918impl std::fmt::Display for CacheCheckError {
10919 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10920 f.write_str(&self.0)
10921 }
10922}
10923
10924impl std::error::Error for CacheCheckError {}
10925
10926fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10928 state
10929 .resource_block
10930 .get_or_insert_with(|| crate::run::ResourceBlock {
10931 cause: crate::run::ResourceBlockCause::Other,
10932 node: node.to_owned(),
10933 kind: "shared build cache".into(),
10934 detail: format!("{error:#}"),
10935 at: Timestamp::now(),
10936 });
10937 if let Err(error) = state.save() {
10938 tracing::warn!("could not persist cache resource block: {error:#}");
10939 }
10940}
10941
10942#[allow(clippy::too_many_arguments)]
10963async fn with_cache_lease<'s, F, Fut>(
10964 state: &'s mut RunState,
10965 cache_dir: Option<&Path>,
10966 node: &str,
10967 seat: &str,
10968 worktree: &Path,
10969 head: &str,
10970 budget: Duration,
10971 context: &str,
10972 wait: &crate::repo_lock::WaitPolicy,
10973 body: F,
10974) -> Result<(Vec<CommandOutcome>, bool)>
10975where
10976 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10977 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10978{
10979 let started = Instant::now();
10982 let home = crate::run::home();
10983 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10984 let guard = if let Some(cache_dir) = cache_dir {
10985 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10986 match acquire_cache_lease(
10987 state,
10988 cache_dir,
10989 &owner,
10990 resource_limit,
10991 node,
10992 context,
10993 &|| wait.cancelled(),
10994 )
10995 .await
10996 .and_then(|g| {
10997 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10998 }) {
10999 Ok(g) => Some(g),
11000 Err(e) => {
11001 record_cache_error(state, node, &e);
11002 return Ok((
11003 vec![CommandOutcome {
11004 command: "(waiting for the shared build cache)".into(),
11005 code: None,
11006 output_tail: e.to_string(),
11007 duration_ms: started.elapsed().as_millis() as u64,
11008 resource_blocked: true,
11009 }],
11010 false,
11011 ));
11012 }
11013 }
11014 } else {
11015 None
11016 };
11017 let _coordination = coordinate_operation(state, worktree, wait).await?;
11018 if let Some(cache_dir) = cache_dir {
11019 let identity = crate::cache::Identity::new(worktree, head);
11020 if let Err(e) = crate::cache::ensure_fresh(
11021 &home,
11022 cache_dir,
11023 &identity,
11024 &state.config.verify.cargo_manifests(),
11025 ) {
11026 record_cache_error(state, node, &e);
11027 state.event(
11028 "verify",
11029 format!(
11030 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11031 worktree.display(),
11032 short(head)
11033 ),
11034 );
11035 if let Some(guard) = guard {
11036 guard.release();
11037 }
11038 return Ok((
11039 vec![CommandOutcome {
11040 command: "(confirming the shared build cache is fresh)".into(),
11041 code: None,
11042 output_tail: e.to_string(),
11043 duration_ms: started.elapsed().as_millis() as u64,
11044 resource_blocked: true,
11045 }],
11046 false,
11047 ));
11048 }
11049 }
11050 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11051 if !timed_out_pids.is_empty() {
11052 wait_for_timed_out_children_to_die(&timed_out_pids).await;
11053 }
11054 drop(_coordination);
11057 if let Some(guard) = guard {
11058 guard.release();
11059 }
11060 Ok((outcomes, retried))
11061}
11062
11063async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11075 wait_for_pids_with(
11076 pids,
11077 crate::proc::pid_alive,
11078 LEASE_RELEASE_POLL,
11079 LEASE_RELEASE_MAX_WAIT,
11080 )
11081 .await;
11082}
11083
11084async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11090 pids: &[u32],
11091 alive: F,
11092 poll: Duration,
11093 max_wait: Duration,
11094) {
11095 let deadline = Instant::now() + max_wait;
11096 loop {
11097 if pids.iter().all(|&pid| !alive(pid)) {
11098 return;
11099 }
11100 if Instant::now() >= deadline {
11101 return;
11102 }
11103 tokio::time::sleep(poll).await;
11104 }
11105}
11106
11107fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11114 outcomes.iter().any(|o| o.resource_blocked)
11115}
11116
11117pub(crate) enum GateFix {
11119 Retry,
11121 Stop,
11124 Defer,
11127}
11128
11129fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11137 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11138 red.peek().is_some()
11139 && red.all(|o| {
11140 !o.resource_blocked
11141 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11142 && !o.output_tail.trim().is_empty()
11143 })
11144}
11145
11146fn e2e_outcome_label(o: &CommandOutcome) -> String {
11150 if o.ok() {
11151 return "pass".to_owned();
11152 }
11153 let reason = if o.build_failed() {
11154 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11155 } else {
11156 format!("FAIL ({:?})", o.code)
11157 };
11158 format!(
11159 "{reason}\n{}",
11160 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11161 )
11162}
11163
11164async fn run_e2e_with_retry(
11172 state: &mut RunState,
11173 shell: &[String],
11174 commands: &[String],
11175 worktree: &Path,
11176 timeout: Duration,
11177 context: &str,
11178) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11179 let (mut e2e, mut timed_out_pids) = run_commands(
11180 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11181 )
11182 .await;
11183 for o in &e2e {
11184 state.event(
11185 "verify",
11186 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11187 );
11188 }
11189 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11193 if verify_retried {
11194 state.event(
11195 "verify",
11196 format!(
11197 "{context}: verify could not build/link, not a test result — retrying once \
11198 before concluding"
11199 ),
11200 );
11201 let retried = run_commands(
11202 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11203 )
11204 .await;
11205 e2e = retried.0;
11206 timed_out_pids.extend(retried.1);
11209 for o in &e2e {
11210 state.event(
11211 "verify",
11212 format!(
11213 "{context}: retry `{}` -> {}",
11214 o.command,
11215 e2e_outcome_label(o)
11216 ),
11217 );
11218 }
11219 }
11220 (e2e, verify_retried, timed_out_pids)
11221}
11222
11223#[allow(clippy::too_many_arguments)]
11239async fn run_commands(
11240 state: &mut RunState,
11241 node: &str,
11242 task: &str,
11243 attempt: usize,
11244 shell: &[String],
11245 commands: &[String],
11246 cwd: &Path,
11247 timeout: Duration,
11248) -> (Vec<CommandOutcome>, Vec<u32>) {
11249 if commands.is_empty() {
11250 return (Vec::new(), Vec::new());
11255 }
11256 let mut out = Vec::new();
11257 let mut timed_out_pids = Vec::new();
11258 let total = commands.len();
11259 for (idx, command) in commands.iter().enumerate() {
11260 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11261 if let Err(e) = state.save() {
11262 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11263 }
11264 let started = Instant::now();
11265 let mut cmd = tokio::process::Command::new(&shell[0]);
11266 cmd.quiet();
11267 for key in crate::worker::INVOCATION_ENV {
11271 cmd.env_remove(key);
11272 }
11273 cmd.args(&shell[1..])
11274 .arg(command)
11275 .current_dir(cwd)
11276 .stdin(std::process::Stdio::null())
11277 .stdout(std::process::Stdio::piped())
11278 .stderr(std::process::Stdio::piped())
11279 .kill_on_drop(true);
11280 let mut outputs = Vec::new();
11281 let protected = (|| -> Result<()> {
11282 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11283 outputs.push(guard);
11284 }
11285 if let Some(cache) = state.config.cache_dir() {
11286 outputs.push(crate::retention::OutputLease::cache(
11287 &crate::run::home(),
11288 &cache,
11289 false,
11290 )?);
11291 }
11292 Ok(())
11293 })();
11294 if let Err(e) = protected {
11295 out.push(CommandOutcome {
11296 command: command.clone(),
11297 code: None,
11298 output_tail: format!("output ownership unavailable: {e:#}"),
11299 duration_ms: 0,
11300 resource_blocked: true,
11301 });
11302 continue;
11303 }
11304 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11305 let (code, body) = match spawned {
11306 Ok(mut running) => {
11307 let pid = running.child().id();
11308 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11309 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11310 let mut paths = vec![cwd.to_owned()];
11311 if let Some(cache) = state.config.cache_dir() {
11312 paths.push(cache);
11313 }
11314 let mut pressure = None;
11315 let status = tokio::select! {
11316 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11317 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11318 pressure = Some(reason); None
11319 }
11320 };
11321 let (code, failure) = match status {
11322 Some(Ok(Ok(status))) => (
11323 status.code(),
11324 status
11325 .code()
11326 .is_none()
11327 .then(|| format!("command terminated: {status}")),
11328 ),
11329 Some(Ok(Err(error))) => {
11330 (None, Some(format!("failed waiting for command: {error}")))
11331 }
11332 Some(Err(_)) => {
11333 if !running.coordinated()
11337 && let Some(pid) = pid
11338 {
11339 timed_out_pids.push(pid);
11340 }
11341 (
11342 None,
11343 Some(format!("timed out after {}s", timeout.as_secs())),
11344 )
11345 }
11346 None => (None, None),
11347 };
11348 running.finish().await;
11349 if let Some(reason) = pressure {
11350 state.resource_block = Some(crate::run::ResourceBlock {
11351 cause: crate::run::ResourceBlockCause::Other,
11352 node: node.to_owned(),
11353 kind: "disk pressure".to_owned(),
11354 detail: reason.clone(),
11355 at: Timestamp::now(),
11356 });
11357 out.push(CommandOutcome {
11358 command: command.clone(),
11359 code: None,
11360 output_tail: reason,
11361 duration_ms: started.elapsed().as_millis() as u64,
11362 resource_blocked: true,
11363 });
11364 break;
11365 }
11366 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11367 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11368 if let Some(failure) = failure {
11369 body.push('\n');
11370 body.push_str(&failure);
11371 }
11372 (code, body)
11373 }
11374 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11375 };
11376 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11377 if !body.is_empty() {
11378 match crate::run::write_verification_artifact(state, task, &body) {
11382 Ok(name) if summary != body => {
11383 summary = format!("[full output: artifacts/{name}]\n{summary}");
11384 }
11385 Ok(_) => {}
11386 Err(e) => {
11387 let warning = format!("could not save the full {task} output: {e:#}");
11388 tracing::warn!("{warning}");
11389 state.event(node, warning);
11390 if summary != body {
11391 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11392 }
11393 }
11394 }
11395 }
11396 out.push(CommandOutcome {
11397 command: command.clone(),
11398 code,
11399 output_tail: summary,
11400 duration_ms: started.elapsed().as_millis() as u64,
11401 resource_blocked: false,
11402 });
11403 }
11404 state.task_finished(task);
11405 if let Err(e) = state.save() {
11406 tracing::warn!("could not persist the end of {task}: {e:#}");
11407 }
11408 (out, timed_out_pids)
11409}
11410
11411fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11423 let repo = repo.display();
11424 match style {
11425 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11426 MergeStyle::Squash => {
11427 let subject = message
11430 .lines()
11431 .next()
11432 .unwrap_or(branch)
11433 .replace(['\\', '"', '$', '`'], "");
11434 format!(
11435 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11436 )
11437 }
11438 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11439 }
11440}
11441
11442const PR_TITLE_MAX: usize = 240;
11453
11454fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11465 let line = queue::first_line(text)?;
11466 let chars: Vec<char> = line.chars().collect();
11467 let (kept, mark) = if chars.len() <= max {
11468 (line.to_owned(), "")
11469 } else if let Some(end) = sentence_end(&chars, max) {
11470 (chars[..end].iter().collect::<String>(), "")
11471 } else {
11472 let room = max.saturating_sub(3);
11473 let cut = if chars[room].is_whitespace() {
11476 room
11477 } else {
11478 chars[..room]
11479 .iter()
11480 .rposition(|c| c.is_whitespace())
11481 .unwrap_or(room)
11482 };
11483 let head: String = chars[..cut].iter().collect();
11484 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11485 (head.to_owned(), "...")
11486 };
11487 if kept.is_empty() {
11488 return None;
11489 }
11490 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11491 return None;
11492 }
11493 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11494 return None;
11495 }
11496 Some(format!("{kept}{mark}"))
11497}
11498
11499fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11503 const MIN: usize = 20;
11504 for i in MIN..max.min(chars.len()) {
11505 if !matches!(chars[i], '.' | '!' | '?') {
11506 continue;
11507 }
11508 let Some(&next) = chars.get(i + 1) else {
11509 continue;
11510 };
11511 if !next.is_whitespace() {
11512 continue;
11513 }
11514 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11515 if after.is_some_and(|c| c.is_ascii_lowercase()) {
11516 continue;
11517 }
11518 let word: String = chars[..i]
11519 .iter()
11520 .rev()
11521 .take_while(|c| !c.is_whitespace())
11522 .collect::<Vec<_>>()
11523 .into_iter()
11524 .rev()
11525 .collect();
11526 let word = word.to_ascii_lowercase();
11527 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11528 continue;
11529 }
11530 let end = chars[..i]
11531 .iter()
11532 .rposition(|c| !c.is_whitespace())
11533 .map_or(i, |p| p + 1);
11534 return Some(end);
11535 }
11536 None
11537}
11538
11539struct PrMessage {
11544 title: String,
11545 body: String,
11546}
11547
11548impl PrMessage {
11549 fn commit_message(&self) -> String {
11553 format!("{}\n\n{}", self.title, self.body)
11554 }
11555}
11556
11557fn title_marker(line: &str) -> Option<&str> {
11559 let line = line.trim();
11560 let head = line.get(..6)?;
11561 head.eq_ignore_ascii_case("title:")
11562 .then(|| line[6..].trim())
11563}
11564
11565fn summary_title(summary: &str) -> Option<String> {
11570 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11571 let raw = title_marker(first)?;
11572 if raw.is_empty() {
11573 return None;
11574 }
11575 let title = queue::title_from(raw, PR_TITLE_MAX);
11576 let lower = title.to_ascii_lowercase();
11577 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11578 return None;
11579 }
11580 Some(title)
11581}
11582
11583const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11585
11586pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11588
11589pub(crate) fn is_review_run(state: &RunState) -> bool {
11593 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11594}
11595
11596fn review_title(state: &RunState) -> Option<String> {
11601 english_subject(
11602 state.reviewed_commits.as_ref()?.first()?,
11603 state.config.graph.github_english(),
11604 )
11605}
11606
11607fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11610 let raw = raw.trim();
11611 if raw.is_empty() {
11612 return None;
11613 }
11614 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11615 return None;
11616 }
11617 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11618 return None;
11619 }
11620 let title = queue::title_from(raw, PR_TITLE_MAX);
11621 let lower = title.to_ascii_lowercase();
11622 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11623 return None;
11624 }
11625 Some(title)
11626}
11627
11628#[derive(Debug, Clone, PartialEq, Eq)]
11631struct BranchFacts {
11632 commits: Vec<(String, String)>,
11634 stat: String,
11636}
11637
11638const STAT_FILE_LINES: usize = 25;
11640const COMMITS_MAX_CHARS: usize = 20_000;
11642
11643async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11646 let commits = git::commit_log(repo, base, branch).await.ok()?;
11647 if commits.is_empty() {
11648 return None;
11649 }
11650 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11651 let lines: Vec<&str> = stat.lines().collect();
11652 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11653 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11654 let more = format!(" ... {omitted} more file(s)");
11655 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11656 kept.push(&more);
11657 kept.push(lines[lines.len() - 1]);
11658 kept.join("\n")
11659 } else {
11660 lines.join("\n")
11661 };
11662 Some(BranchFacts { commits, stat })
11663}
11664
11665fn markdown_safe(text: &str) -> String {
11668 text.replace("</details>", "</details>")
11669 .replace("\x60\x60\x60", "~~~")
11670}
11671
11672fn neutral_title(state: &RunState, winner: char) -> String {
11673 format!(
11674 "chore: land candidate {} of run {}",
11675 winner.to_ascii_uppercase(),
11676 state.id
11677 )
11678}
11679
11680pub(crate) async fn repair_landing_gate(
11683 state: &mut RunState,
11684 winner: &Candidate,
11685 outcomes: &[CommandOutcome],
11686) -> Result<GateFix> {
11687 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11688 return Ok(GateFix::Stop);
11689 }
11690 let mut runner = Runner {
11691 roles: state.config.resolve_roles()?,
11692 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11693 state: state.clone(),
11694 pause: Pause::new(),
11695 interrupt: Pause::new(),
11696 };
11697 let result = runner.gate_fix_round(winner, outcomes).await;
11698 *state = runner.state;
11699 result
11700}
11701
11702pub(crate) async fn verify_current_head(
11705 state: &mut RunState,
11706 winner: &Candidate,
11707 wait: &crate::repo_lock::WaitPolicy,
11708) -> Result<bool> {
11709 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11710 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11712 state
11713 .verification_history
11714 .push(crate::run::VerificationEvidence {
11715 superseded_by: Some(head.clone()),
11716 archived_at: Some(Timestamp::now()),
11717 fixes_spent: Some(state.gate_fixes.len()),
11718 gate_head: None,
11719 gate_ran: false,
11720 gate: Vec::new(),
11721 e2e: state.head_e2e.take(),
11722 });
11723 }
11724 if let Some(e) = &state.head_e2e
11725 && e.head == head
11726 {
11727 if e.passed {
11728 return Ok(true);
11729 }
11730 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11731 state.status = RunStatus::Blocked;
11732 state.save()?;
11733 return Ok(false);
11734 }
11735 } else if state.reviews.last().is_some_and(|r| {
11736 r.verified_head.as_deref() == Some(head.as_str())
11737 && (r.e2e_status() == E2eStatus::Passed
11738 || (r.e2e_status() == E2eStatus::NotConfigured
11739 && state.config.verify.e2e.is_empty()))
11740 }) || state
11741 .gate_repair_e2e
11742 .as_ref()
11743 .is_some_and(|e| e.head == head && e.passed)
11744 {
11745 return Ok(true);
11746 }
11747 if let Some(e) = &state.gate_repair_e2e
11748 && e.head == head
11749 && !e.passed
11750 && !e.outcomes.is_empty()
11751 && !verify_inconclusive(&e.outcomes)
11752 {
11753 state.status = RunStatus::Blocked;
11754 state.save()?;
11755 return Ok(false);
11756 }
11757 state.head_e2e = Some(crate::run::GateRepairE2e {
11758 head: head.clone(),
11759 passed: false,
11760 outcomes: Vec::new(),
11761 });
11762 state.save()?;
11763 let commands = state.config.verify.e2e.clone();
11764 let shell = state.config.shell();
11765 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11766 let cache_dir = state.config.cache_dir();
11767 let outcomes = if commands.is_empty() {
11768 Vec::new()
11769 } else {
11770 let (outcomes, _) = with_cache_lease(
11771 state,
11772 cache_dir.as_deref(),
11773 "e2e",
11774 "e2e",
11775 &winner.worktree,
11776 &head,
11777 timeout,
11778 "current head e2e",
11779 wait,
11780 |state, budget| async move {
11781 run_e2e_with_retry(
11782 state,
11783 &shell,
11784 &commands,
11785 &winner.worktree,
11786 budget,
11787 "current head e2e",
11788 )
11789 .await
11790 },
11791 )
11792 .await?;
11793 outcomes
11794 };
11795 let unresolved = verify_inconclusive(&outcomes);
11796 let passed = !unresolved
11797 && outcomes.iter().all(CommandOutcome::ok)
11798 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11799 state.head_e2e = Some(crate::run::GateRepairE2e {
11800 head: head.clone(),
11801 passed,
11802 outcomes,
11803 });
11804 if !passed && !unresolved {
11805 state.status = RunStatus::Blocked;
11806 state.event(
11807 "e2e",
11808 format!("verify.e2e failed on head {head}; nothing published"),
11809 );
11810 }
11811 state.save()?;
11812 Ok(passed)
11813}
11814
11815pub(crate) async fn check_gate_repair_e2e(
11818 state: &mut RunState,
11819 winner: &Candidate,
11820 wait: &crate::repo_lock::WaitPolicy,
11821) -> Result<GateFix> {
11822 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11823 return Ok(GateFix::Retry);
11824 }
11825 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11826 if state
11827 .gate_repair_e2e
11828 .as_ref()
11829 .is_some_and(|r| r.head == head && r.passed)
11830 {
11831 return Ok(GateFix::Retry);
11832 }
11833 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11834 head: head.clone(),
11835 passed: false,
11836 outcomes: Vec::new(),
11837 });
11838 state.save()?;
11839 let commands = state.config.verify.e2e.clone();
11840 let outcomes = if commands.is_empty() {
11841 Vec::new()
11842 } else {
11843 let shell = state.config.shell();
11844 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11845 let cache_dir = state.config.cache_dir();
11846 let (outcomes, _) = with_cache_lease(
11847 state,
11848 cache_dir.as_deref(),
11849 "e2e",
11850 "e2e",
11851 &winner.worktree,
11852 &head,
11853 timeout,
11854 "gate repair e2e",
11855 wait,
11856 |state, budget| async move {
11857 run_e2e_with_retry(
11858 state,
11859 &shell,
11860 &commands,
11861 &winner.worktree,
11862 budget,
11863 "gate repair e2e",
11864 )
11865 .await
11866 },
11867 )
11868 .await?;
11869 outcomes
11870 };
11871 let deferred = verify_inconclusive(&outcomes);
11872 let passed = outcomes.iter().all(CommandOutcome::ok)
11873 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11874 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11875 head,
11876 passed,
11877 outcomes,
11878 });
11879 state.save()?;
11880 if deferred {
11881 Ok(GateFix::Defer)
11882 } else if !passed {
11883 state.event(
11884 "gate",
11885 "verify.e2e failed after gate repair; nothing published",
11886 );
11887 Ok(GateFix::Stop)
11888 } else {
11889 Ok(GateFix::Retry)
11890 }
11891}
11892
11893pub(crate) async fn run_landing_gate(
11895 state: &mut RunState,
11896 winner: &Candidate,
11897 wait: &crate::repo_lock::WaitPolicy,
11898) -> Result<Vec<CommandOutcome>> {
11899 let shell = state.config.shell();
11900 let gate_commands = state.config.verify.gate.clone();
11901 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11902 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11903 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
11904 Vec::new()
11905 } else {
11906 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11907 let cache_dir = state.config.cache_dir();
11908 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11909 let (outcomes, _) = with_cache_lease(
11910 state,
11911 cache_dir.as_deref(),
11912 "gate",
11913 "gate",
11914 &winner.worktree,
11915 &head,
11916 timeout,
11917 "final gate",
11918 wait,
11919 |state, budget| {
11920 let shell = shell.clone();
11921 let gate_commands = gate_commands.clone();
11922 let winner = winner.clone();
11923 async move {
11924 Runner::run_pre_gate(state, &winner).await;
11925 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
11927 let (outcomes, timed_out_pids) = run_commands(
11928 state,
11929 "gate",
11930 "gate",
11931 0,
11932 &shell,
11933 &gate_commands,
11934 &winner.worktree,
11935 budget,
11936 )
11937 .await;
11938 (outcomes, false, timed_out_pids)
11939 }
11940 },
11941 )
11942 .await?;
11943 outcomes
11944 };
11945 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
11946 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
11947 let mut outcomes = outcomes;
11948 outcomes.push(CommandOutcome {
11949 command: "gate head stability".into(), code: None,
11950 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
11951 duration_ms: 0, resource_blocked: false,
11952 });
11953 return Ok(outcomes);
11954 }
11955 if outcomes.is_empty() {
11956 state.event("gate", "no gate commands configured; not tested");
11961 }
11962 for o in &outcomes {
11963 state.event(
11964 "gate",
11965 format!(
11966 "`{}` -> {}",
11967 o.command,
11968 if o.ok() {
11969 "pass".to_owned()
11970 } else {
11971 format!(
11972 "FAIL ({:?})\n{}",
11973 o.code,
11974 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11975 )
11976 }
11977 ),
11978 );
11979 }
11980 Ok(outcomes)
11981}
11982
11983pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11989 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11990 ""
11991 } else {
11992 pr_title
11993 }
11994}
11995
11996pub fn landing_subject_source(state: &RunState) -> String {
12000 if is_review_run(state) {
12001 let winner = state.candidates.first().map_or('A', |c| c.label);
12002 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12003 }
12004 state.instruction.clone()
12005}
12006
12007fn summary_without_title(summary: &str) -> String {
12010 let mut lines = summary.trim().lines().peekable();
12011 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12012 lines.next();
12013 }
12014 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12015}
12016
12017#[cfg(test)]
12029fn pr_message(state: &RunState, winner: char) -> PrMessage {
12030 pr_message_with(state, winner, None)
12031}
12032
12033#[cfg(test)]
12036fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12037 let raw = pr_message_raw(
12038 state,
12039 winner,
12040 facts,
12041 crate::pr_outcome::FinalHead::Unreported,
12042 ChangeNote::NotAsked,
12043 );
12044 let id = crate::scrub::Identity::current();
12045 PrMessage {
12046 title: crate::scrub::scrub(&raw.title, &id),
12047 body: crate::scrub::scrub(&raw.body, &id),
12048 }
12049}
12050
12051#[derive(Debug, Clone, Copy)]
12053enum ChangeNote<'a> {
12054 NotAsked,
12056 Summary(&'a crate::run::ChangeSummary),
12058 Unavailable(&'a str),
12060}
12061
12062fn pr_message_raw(
12063 state: &RunState,
12064 winner: char,
12065 facts: Option<&BranchFacts>,
12066 head: crate::pr_outcome::FinalHead<'_>,
12067 change: ChangeNote<'_>,
12068) -> PrMessage {
12069 if !matches!(change, ChangeNote::NotAsked) {
12070 let (title, mut body) = match change {
12071 ChangeNote::Summary(c) => (
12072 c.title.clone(),
12073 format!("## Summary\n\n{}\n\n", c.body.trim()),
12074 ),
12075 ChangeNote::Unavailable(why) => (
12076 neutral_title(state, winner),
12077 format!(
12078 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12079 ),
12080 ),
12081 ChangeNote::NotAsked => unreachable!(),
12082 };
12083 body.push_str(&public_pr_evidence(state, head));
12084 body.push_str(&format!(
12085 "\n---\nmagi:run/{} magi:candidate-{}\n",
12086 state.id,
12087 winner.to_ascii_lowercase()
12088 ));
12089 return PrMessage { title, body };
12090 }
12091 pr_preview(state, winner, facts, head)
12092}
12093
12094fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12097 let mut body = crate::pr_outcome::review_rounds(state);
12098 body.push_str(&crate::pr_outcome::final_verification(state, head));
12099 let open = state.last_round_findings();
12100 if !open.is_empty() {
12101 body.push_str("## Open review findings\n\n");
12102 for f in open {
12103 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12104 }
12105 body.push('\n');
12106 }
12107 if let Some(c) = state.change_summary.as_ref()
12108 && head == crate::pr_outcome::FinalHead::At(&c.head)
12109 {
12110 body.push_str(&format!(
12111 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12112 c.base, c.head
12113 ));
12114 }
12115 body
12116}
12117
12118fn pr_preview(
12120 state: &RunState,
12121 winner: char,
12122 facts: Option<&BranchFacts>,
12123 head: crate::pr_outcome::FinalHead<'_>,
12124) -> PrMessage {
12125 let summary = state
12126 .candidates
12127 .iter()
12128 .find(|c| c.label == winner)
12129 .map(|c| c.summary.as_str())
12130 .unwrap_or_default();
12131 let review = is_review_run(state);
12134 let title = if review {
12135 facts
12136 .and_then(|f| {
12137 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12138 })
12139 .or_else(|| review_title(state))
12140 .or_else(|| {
12141 state
12142 .candidates
12143 .iter()
12144 .find(|c| c.label == winner)
12145 .filter(|c| !c.branch.starts_with("magi/"))
12146 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12147 })
12148 .unwrap_or_else(|| neutral_title(state, winner))
12149 } else {
12150 summary_title(summary).unwrap_or_else(|| {
12151 english_title(
12152 &state.instruction,
12153 PR_TITLE_MAX,
12154 state.config.graph.github_english(),
12155 )
12156 .unwrap_or_else(|| neutral_title(state, winner))
12157 })
12158 };
12159
12160 let mut body = String::new();
12161 let what = summary_without_title(summary);
12162 if !what.is_empty() {
12163 body.push_str("## Summary\n\n");
12164 body.push_str(&what);
12165 body.push_str("\n\n");
12166 }
12167
12168 body.push_str(&crate::pr_outcome::review_rounds(state));
12170 body.push_str(&crate::pr_outcome::final_verification(state, head));
12171 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12172
12173 let open = state.open_findings();
12174 if !open.is_empty() {
12175 body.push_str("## Open review findings\n\n");
12176 for f in &open {
12177 body.push_str(&format!(
12178 "- `{}` [{:?}] {}\n",
12179 f.id, f.severity, "Context retained in local review records."
12180 ));
12181 }
12182 body.push('\n');
12183 }
12184
12185 if let Some(fix) = fix
12186 && !fix.rejected.is_empty()
12187 {
12188 body.push_str("## Declined by the fixer\n\n");
12189 for r in &fix.rejected {
12190 body.push_str(&format!(
12191 "- `{}`: {}\n",
12192 r.id, "Reason retained in local review records."
12193 ));
12194 }
12195 body.push('\n');
12196 }
12197
12198 if review {
12199 body.push_str("## Commits under review\n\n");
12201 if let Some(facts) = facts {
12202 let mut left = COMMITS_MAX_CHARS;
12203 for (i, (subject, text)) in facts.commits.iter().enumerate() {
12204 let mut entry = format!("- {}\n", markdown_safe(subject));
12205 for l in markdown_safe(text).lines() {
12206 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
12207 }
12208 if left == 0 {
12209 body.push_str(&format!(
12210 "- ... {} more commit(s)\n",
12211 facts.commits.len() - i
12212 ));
12213 break;
12214 }
12215 if entry.len() > left {
12216 let mut end = left;
12219 while !entry.is_char_boundary(end) {
12220 end -= 1;
12221 }
12222 entry.truncate(end);
12223 entry.push_str("\n ... (truncated)\n");
12224 left = 0;
12225 } else {
12226 left -= entry.len();
12227 }
12228 body.push_str(&entry);
12229 }
12230 if !facts.stat.trim().is_empty() {
12231 body.push_str(&format!(
12232 "\n## Diff stat\n\n```\n{}\n```\n",
12233 markdown_safe(facts.stat.trim())
12234 ));
12235 }
12236 } else {
12237 match &state.reviewed_commits {
12238 Some(subjects) => {
12239 for s in subjects {
12240 body.push_str(&format!("- {}\n", s.trim()));
12241 }
12242 }
12243 None => {
12244 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12247 body.push_str(rest.trim());
12248 body.push('\n');
12249 }
12250 }
12251 }
12252 } else {
12253 let task = state.instruction.trim();
12254 let task = if task.is_empty() {
12255 "(empty task)"
12256 } else {
12257 task
12258 };
12259 body.push_str(&format!(
12260 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12261 task.replace("</details>", "</details>")
12262 ));
12263 }
12264
12265 body.push_str(&format!(
12266 "\n---\nmagi:run/{} magi:candidate-{}\n",
12267 state.id,
12268 winner.to_ascii_lowercase()
12269 ));
12270
12271 PrMessage { title, body }
12272}
12273
12274fn seeded_instruction(state: &RunState) -> String {
12277 let mut inputs = Vec::new();
12278 if let Some(start) = &state.sources.start_from {
12279 inputs.push(format!(
12280 "Start from `{}` at `{}` (including ancestry).",
12281 start.token, start.sha
12282 ));
12283 }
12284 for pick in &state.sources.pick_commits {
12285 inputs.push(format!(
12286 "Pick `{}` at `{}` (single commit).",
12287 pick.token, pick.sha
12288 ));
12289 }
12290 if inputs.is_empty() {
12291 state.instruction.clone()
12292 } else {
12293 format!(
12294 "{}\n\n# Explicit source inputs already prepared\n\n{}",
12295 state.instruction,
12296 inputs.join("\n")
12297 )
12298 }
12299}
12300
12301async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12305 let base = &state.base_branch;
12306 let mut against = base.clone();
12307 if mode == MergeMode::Pr {
12308 let remote = &state.config.merge.remote;
12312 let tracking = format!("{remote}/{base}");
12313 let fetched = git::fetch(repo, remote, base).await;
12314 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12315 against = tracking;
12316 } else {
12317 return false;
12318 }
12319 }
12320 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12321}
12322
12323fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12326 let mut detail = format!(
12327 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12328 nothing to open a pull request for"
12329 );
12330 detail.push_str(
12331 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12332 );
12333 detail
12334}
12335
12336#[derive(Debug, PartialEq, Eq)]
12339enum PrPlan {
12340 Create,
12341 Adopt { url: String, title: String },
12342 Stop(String),
12343}
12344
12345fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12348 match found {
12349 Ok(land::OpenPr::None) => PrPlan::Create,
12350 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12351 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12352 "several open pull requests exist for this branch, not picking one: {}",
12353 urls.join(" ")
12354 )),
12355 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12356 }
12357}
12358
12359pub(crate) async fn gh_pr_create(
12361 cwd: &Path,
12362 base: &str,
12363 head: &str,
12364 title: &str,
12365 body: &str,
12366) -> Result<String> {
12367 let out = tokio::process::Command::new("gh")
12368 .args([
12369 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12370 ])
12371 .current_dir(cwd)
12372 .quiet()
12373 .stdin(std::process::Stdio::null())
12374 .output()
12375 .await
12376 .context("spawn gh")?;
12377 if out.status.success() {
12378 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12379 } else {
12380 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12381 }
12382}
12383
12384async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12386 match std::fs::symlink_metadata(path) {
12387 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12388 git::worktree_remove_clean(repo, path).await?;
12391 return Ok(());
12392 }
12393 Err(e) => return Err(e.into()),
12394 Ok(_) => {}
12395 }
12396 if !git::is_clean(path).await? {
12397 bail!(
12398 "worktree has uncommitted work; refusing removal: {}",
12399 path.display()
12400 );
12401 }
12402 if !git::worktree_remove_clean(repo, path).await? {
12403 bail!("worktree removal refused: {}", path.display());
12404 }
12405 Ok(())
12406}
12407
12408pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12417 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12418 bail!("refusing fold: {reason}");
12419 }
12420 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12421 bail!("candidate preservation incomplete; refusing cleanup");
12422 }
12423 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12427 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12428 let _driver = crate::driver::maintenance(home, &state.id)?;
12429 let mut output_guards = Vec::new();
12430 for entry in std::fs::read_dir(state.worktree_root())
12431 .into_iter()
12432 .flatten()
12433 .flatten()
12434 {
12435 if entry.path().join(".git").is_file() {
12436 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12437 }
12438 }
12439 let repo = state.repo.clone();
12440 let wait = repository_wait(state, None);
12443 let _coordination = match coordinate_operation(state, &repo, &wait).await {
12444 Ok(lease) => lease,
12445 Err(e) => {
12446 state.save_under(home)?;
12447 return Err(e);
12448 }
12449 };
12450 let _source_outputs = if state.status == RunStatus::Merged
12453 && state.merge.as_ref().is_some_and(|merge| merge.ok)
12454 {
12455 verify_merged_fold_sources(state, drop_winner, home).await?
12456 } else {
12457 Vec::new()
12458 };
12459 let root = state.worktree_root();
12460 let winner = state.tally.as_ref().map(|t| t.winner);
12461 let mut removed = Vec::new();
12462
12463 for i in 0..state.candidates.len() {
12464 let c = state.candidates[i].clone();
12465 let is_winner = Some(c.label) == winner;
12466 if is_winner && !drop_winner {
12467 continue;
12468 }
12469 let existed = c.worktree.exists();
12470 remove_preserved_worktree(&repo, &c.worktree).await?;
12473 if existed {
12474 removed.push(c.worktree.to_string_lossy().into_owned());
12475 }
12476 let handed_over = state.released_branches.contains(&c.branch);
12479 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12480 if !git::branch_delete(&repo, &c.branch).await? {
12481 bail!("candidate branch removal refused");
12482 }
12483 removed.push(c.branch.clone());
12484 }
12485 state.candidates[i].folded = true;
12486 }
12487
12488 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12489 let path = name.path();
12490 let keep = !drop_winner
12491 && winner.is_some_and(|w| {
12492 path.file_name()
12493 .is_some_and(|n| n == format!("cand-{w}").as_str())
12494 });
12495 if keep {
12496 continue;
12497 }
12498 remove_preserved_worktree(&repo, &path).await?;
12499 removed.push(path.to_string_lossy().into_owned());
12500 }
12501
12502 remove_if_empty(&root);
12511
12512 if state.enabled_worktree_config && drop_winner {
12513 git::release_worktree_config(&repo).await.ok();
12517 state.enabled_worktree_config = false;
12518 }
12519 state.save_under(home)?;
12520 Ok(removed)
12521}
12522
12523async fn verify_merged_fold_sources(
12524 state: &RunState,
12525 drop_winner: bool,
12526 home: &Path,
12527) -> Result<Vec<std::fs::File>> {
12528 let winner = state.tally.as_ref().map(|t| t.winner);
12529 let mut heads = BTreeSet::new();
12530 for candidate in &state.candidates {
12531 if Some(candidate.label) == winner && !drop_winner {
12532 continue;
12533 }
12534 anyhow::ensure!(
12535 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12536 "candidate recovery source is outside the inventoried run root: {}",
12537 candidate.worktree.display()
12538 );
12539 match std::fs::symlink_metadata(&candidate.worktree) {
12540 Ok(_) => {}
12541 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12542 Err(error) => return Err(error.into()),
12543 }
12544 if !state.released_branches.contains(&candidate.branch)
12545 && git::branch_exists(&state.repo, &candidate.branch).await?
12546 {
12547 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12548 }
12549 }
12550 let mut cargo_guards = Vec::new();
12551 let entries = match std::fs::read_dir(state.worktree_root()) {
12552 Ok(entries) => Some(entries),
12553 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12554 Err(error) => return Err(error.into()),
12555 };
12556 for entry in entries.into_iter().flatten() {
12557 let path = entry?.path();
12558 let metadata = std::fs::symlink_metadata(&path)?;
12559 anyhow::ensure!(
12560 metadata.is_dir() && !metadata.file_type().is_symlink(),
12561 "non-directory or symlink recovery source retained: {}",
12562 path.display()
12563 );
12564 if !drop_winner
12565 && winner.is_some_and(|w| {
12566 path.file_name()
12567 .is_some_and(|name| name == format!("cand-{w}").as_str())
12568 })
12569 {
12570 continue;
12571 }
12572 anyhow::ensure!(
12573 path.join(".git").is_file(),
12574 "unregistered recovery source retained: {}",
12575 path.display()
12576 );
12577 anyhow::ensure!(
12578 git::git(
12579 &path,
12580 &[
12581 "status",
12582 "--porcelain",
12583 "--untracked-files=all",
12584 "--ignored",
12585 "--",
12586 ".",
12587 ":(exclude)target"
12588 ]
12589 )
12590 .await?
12591 .is_empty(),
12592 "source changes or ignored recovery files retained: {}",
12593 path.display()
12594 );
12595 match std::fs::symlink_metadata(path.join("target")) {
12596 Ok(_) => {
12597 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12598 }
12599 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12600 Err(error) => return Err(error.into()),
12601 }
12602 for checkpoint in [
12603 "rebase-merge",
12604 "rebase-apply",
12605 "sequencer",
12606 "CHERRY_PICK_HEAD",
12607 "MERGE_HEAD",
12608 "REBASE_HEAD",
12609 "REVERT_HEAD",
12610 "BISECT_LOG",
12611 "AUTO_MERGE",
12612 "MERGE_MSG",
12613 ] {
12614 let location = git::git(
12615 &path,
12616 &[
12617 "rev-parse",
12618 "--path-format=absolute",
12619 "--git-path",
12620 checkpoint,
12621 ],
12622 )
12623 .await?;
12624 anyhow::ensure!(
12625 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12626 if e.kind() == std::io::ErrorKind::NotFound),
12627 "Git recovery checkpoint retained: {checkpoint}"
12628 );
12629 }
12630 heads.insert(git::rev_parse(&path, "HEAD").await?);
12631 }
12632 crate::fold_evidence::preserve(state, home, &heads).await?;
12633
12634 Ok(cargo_guards)
12635}
12636
12637fn remove_if_empty(dir: &Path) {
12648 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12649 std::fs::remove_dir(dir).ok();
12650 }
12651}
12652
12653pub fn worst_open(state: &RunState) -> Option<Severity> {
12655 state
12656 .reviews
12657 .last()?
12658 .reviews
12659 .iter()
12660 .flat_map(|r| r.findings.iter())
12661 .map(|f| f.severity)
12662 .max()
12663}
12664
12665async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12669 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12670 git::acquire_worktree_config(repo).await?;
12671 state.enabled_worktree_config = true;
12672 }
12673 Ok(())
12674}
12675
12676#[cfg(test)]
12677mod tests {
12678 #[tokio::test]
12679 async fn launcher_base_fetch_requires_remote_success() {
12680 let tmp = tempfile::tempdir().unwrap();
12681 let repo = tmp.path().join("repo");
12682 let remote = tmp.path().join("remote");
12683 std::fs::create_dir(&repo).unwrap();
12684 std::fs::create_dir(&remote).unwrap();
12685 git::git(&remote, &["init", "--bare", "-b", "main"])
12686 .await
12687 .unwrap();
12688 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12689 git::git(
12690 &repo,
12691 &[
12692 "-c",
12693 "user.name=Fixture",
12694 "-c",
12695 "user.email=fixture@example.invalid",
12696 "commit",
12697 "--allow-empty",
12698 "-m",
12699 "fixture",
12700 ],
12701 )
12702 .await
12703 .unwrap();
12704 git::git(
12705 &repo,
12706 &["remote", "add", "origin", remote.to_str().unwrap()],
12707 )
12708 .await
12709 .unwrap();
12710 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12711 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12712 assert_eq!(
12713 resolve_base(&repo, "main", "origin", None)
12714 .await
12715 .unwrap()
12716 .commit,
12717 head
12718 );
12719 #[cfg(unix)]
12720 {
12721 let policy: crate::config::Coordination = toml::from_str(
12722 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12723 ).unwrap();
12724 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12725 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12726 let held = crate::repo_lock::SeatLease::operation_with(
12727 &repo,
12728 Some(&policy),
12729 &crate::repo_lock::WaitPolicy::default(),
12730 )
12731 .await
12732 .unwrap();
12733 assert!(held.is_some());
12734 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12735 tokio::pin!(fetch);
12736 assert!(
12737 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12738 .await
12739 .is_err()
12740 );
12741 assert!(
12742 !repo.join(".git/FETCH_HEAD").exists(),
12743 "no fetch before admission"
12744 );
12745 drop(held);
12746 let fetched = fetch.await.unwrap();
12747 assert_eq!(fetched.commit, head);
12748 assert_eq!(fetched.remote, "origin");
12749 assert_eq!(fetched.branch, "main");
12750 }
12751 git::git(
12753 &repo,
12754 &[
12755 "remote",
12756 "set-url",
12757 "origin",
12758 tmp.path().join("missing").to_str().unwrap(),
12759 ],
12760 )
12761 .await
12762 .unwrap();
12763 let error = resolve_base(&repo, "main", "origin", None)
12764 .await
12765 .unwrap_err()
12766 .to_string();
12767 assert!(error.contains("cannot read origin/main"));
12768 assert!(error.contains("refusing to branch off"));
12769 }
12770
12771 #[test]
12772 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12773 let leaked = vec!["chore(deps): update a crate".to_owned()];
12774 let own = "fix(daemon): apply a chosen action";
12775 assert!(should_retitle("", own, &leaked));
12776 assert!(should_retitle(
12777 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12778 own,
12779 &leaked
12780 ));
12781 assert!(should_retitle(
12782 "chore: land candidate A of run 1",
12783 own,
12784 &leaked
12785 ));
12786 assert!(should_retitle(
12787 "magi: candidate A (uncommitted work)",
12788 own,
12789 &leaked
12790 ));
12791 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12792 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12793 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12794 }
12795
12796 #[test]
12797 fn pr_merge_plan_creates_adopts_or_stops() {
12798 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12799 assert_eq!(
12800 pr_merge_plan(Ok(land::OpenPr::One {
12801 url: "u".into(),
12802 title: "t".into()
12803 })),
12804 PrPlan::Adopt {
12805 url: "u".into(),
12806 title: "t".into()
12807 }
12808 );
12809 let PrPlan::Stop(many) =
12810 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12811 else {
12812 panic!("many must stop");
12813 };
12814 assert!(many.contains('a') && many.contains('b'));
12815 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12816 panic!("a failed lookup must stop");
12817 };
12818 assert!(err.contains("bad token"));
12819 }
12820
12821 use super::*;
12822 use crate::run::GateStatus;
12823 use std::collections::BTreeMap;
12824 use std::time::Duration;
12825
12826 fn conductor() -> AgentSpec {
12827 AgentSpec {
12828 id: "conductor".to_owned(),
12829 kind: crate::config::AgentKind::Command,
12830 model: None,
12831 command: vec!["true".to_owned()],
12832 extra_args: Vec::new(),
12833 env: BTreeMap::new(),
12834 prompt_delivery: None,
12835 max_turn_secs: None,
12836 }
12837 }
12838
12839 fn spec(id: &str) -> AgentSpec {
12840 AgentSpec {
12841 id: id.to_owned(),
12842 kind: crate::config::AgentKind::Command,
12843 model: None,
12844 command: vec!["true".to_owned()],
12845 extra_args: Vec::new(),
12846 env: BTreeMap::new(),
12847 prompt_delivery: None,
12848 max_turn_secs: None,
12849 }
12850 }
12851
12852 fn ids(xs: &[&str]) -> BTreeSet<String> {
12853 xs.iter().map(|s| (*s).to_owned()).collect()
12854 }
12855
12856 #[test]
12857 fn pick_successor_skips_an_agent_another_seat_holds() {
12858 let roster = [spec("a"), spec("b"), spec("c")];
12860 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12861 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12862 }
12863
12864 #[test]
12865 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12866 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12868 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12869 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12870 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12871 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12872 }
12873
12874 #[test]
12875 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12876 let roster = [spec("a"), spec("b")];
12877 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12878 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12879 let carried = ids(&["b"]);
12880 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12881 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12882 }
12883
12884 #[test]
12885 fn pick_successor_returns_none_without_any_untried_successor() {
12886 let roster = [spec("a"), spec("b")];
12887 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12888 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12889 let carried = ids(&["a"]);
12890 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12891 }
12892
12893 #[test]
12894 fn pick_successor_rescue_avoids_another_seats_occupant() {
12895 let roster = [spec("a"), spec("b"), spec("c")];
12897 let carried = ids(&["b", "c"]);
12898 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12899 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12900 }
12901
12902 #[test]
12903 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12904 let roster = [spec("a"), spec("b"), spec("c")];
12905 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12906 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12907 }
12908
12909 #[test]
12910 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12911 let roster = [spec("a"), spec("b"), spec("c")];
12912 let failed = ids(&["a", "b", "c"]);
12913 let mut tried = ids(&["b"]);
12915 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12916 assert_eq!(first.id, "a");
12917 tried.insert(first.id.clone());
12918 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12919 assert_eq!(second.id, "c");
12920 tried.insert(second.id.clone());
12921 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12922 }
12923
12924 #[test]
12925 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12926 let roster = [spec("a"), spec("b")];
12927 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12928 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12929 }
12930
12931 #[test]
12932 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12933 let roster = [spec("a"), spec("b"), spec("c")];
12934 let h = SeatHistory {
12935 failed: ids(&["a"]),
12936 last_ok: Some("b".to_owned()),
12937 last_fail: None,
12938 };
12939 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12940 let gone = SeatHistory {
12942 last_ok: Some("zzz".to_owned()),
12943 ..h.clone()
12944 };
12945 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12946 let failed = SeatHistory {
12947 failed: ids(&["a", "b"]),
12948 last_ok: Some("b".to_owned()),
12949 last_fail: None,
12950 };
12951 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12952 }
12953
12954 #[test]
12955 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12956 let first = SeatState::new("review-1", "alpha", 7);
12957 let next = handover_seat("review-1", "gamma", 7);
12958 assert_ne!(first.claude_session, next.claude_session);
12959 }
12960
12961 #[test]
12962 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12963 let mut state = state_with_summary("x", "y");
12964 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12965 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12966 assert_ne!(a.claude_session, b.claude_session);
12967 }
12968
12969 #[test]
12970 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12971 let roster = [spec("a"), spec("b")];
12972 let h = SeatHistory {
12973 failed: ids(&["a", "b"]),
12974 ..SeatHistory::default()
12975 };
12976 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12977 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12978 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12979 }
12980
12981 #[test]
12988 fn failure_signature_ignores_numbers_and_paths() {
12989 assert_eq!(
12990 failure_signature("exited with Some(2) and no usable output"),
12991 failure_signature("exited with Some(137) and no usable output")
12992 );
12993 assert_eq!(
12994 failure_signature("process exited with no usable output (exit code 2)"),
12995 failure_signature("process exited with no usable output (exit code 137)")
12996 );
12997 for (old, new) in [
12998 (
12999 "exited with Some(2) and no usable output",
13000 "process exited with no usable output (exit code 2)",
13001 ),
13002 (
13003 "exited with None and no usable output",
13004 "process exited with no usable output (exit code unavailable)",
13005 ),
13006 ] {
13007 let previous = FailClass::Other(failure_signature(old));
13008 let current = FailClass::Other(failure_signature(new));
13009 assert_eq!(previous, current, "resumption retains the failure class");
13010 assert!(!should_hand_over(Some(&previous), ¤t));
13011 }
13012 assert_eq!(
13013 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13014 failure_signature("cannot open /var/x.txt: denied")
13015 );
13016 assert_ne!(failure_signature("boom"), failure_signature("bang"));
13017 }
13018
13019 #[test]
13020 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13021 let other = FailClass::Other("x".into());
13022 assert!(should_hand_over(None, &FailClass::Quota));
13023 assert!(should_hand_over(Some(&other), &FailClass::Quota));
13024 assert!(should_hand_over(
13025 Some(&FailClass::Timeout),
13026 &FailClass::Timeout
13027 ));
13028 assert!(should_hand_over(None, &other));
13029 assert!(!should_hand_over(Some(&other), &other));
13030 assert!(should_hand_over(
13031 Some(&other),
13032 &FailClass::Other("y".into())
13033 ));
13034 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13036 assert!(should_hand_over(Some(&FailClass::Quota), &other));
13037 }
13038
13039 #[test]
13040 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13041 let a = SeatState::new("judge-1", "alpha", 7);
13042 let b = handover_seat("judge-1", "beta", 7);
13043 assert_ne!(a.claude_session, b.claude_session);
13044 assert_eq!(b.turns, 0);
13045 }
13046
13047 #[test]
13048 fn a_timeout_is_classified_apart_from_other_failures() {
13049 assert_eq!(
13050 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13051 Some(FailClass::Timeout)
13052 );
13053 assert!(matches!(
13054 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13055 Some(FailClass::Other(_))
13056 ));
13057 }
13058
13059 #[test]
13060 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13061 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13062 let tried = BTreeSet::from(["beta".to_owned()]);
13063 let next = next_untried_in_roster(&roster, 1, &tried);
13066 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13067 }
13068
13069 #[test]
13070 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13071 let roster = vec![spec("alpha"), spec("beta")];
13072 let tried = BTreeSet::from(["beta".to_owned()]);
13073 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13077 }
13078
13079 #[test]
13080 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13081 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13082 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13083 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13087 }
13088
13089 #[test]
13090 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13091 let roster = vec![spec("a"), spec("a"), spec("b")];
13092 let tried = BTreeSet::from(["a".to_owned()]);
13093 let next = next_untried_in_roster(&roster, 0, &tried);
13094 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13095 }
13096
13097 #[test]
13098 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13099 let roster = vec![spec("a"), spec("b")];
13100 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13101 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13102 }
13103
13104 #[test]
13105 fn remove_if_empty_only_ever_takes_a_bare_directory() {
13106 let dir = tempfile::tempdir().unwrap();
13107 let bay = dir.path().join("ffff");
13108
13109 remove_if_empty(&bay);
13111 assert!(!bay.exists());
13112
13113 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13116 remove_if_empty(&bay);
13117 assert!(bay.exists(), "non-empty directory must survive");
13118
13119 std::fs::remove_dir(bay.join("cand-A")).unwrap();
13121 remove_if_empty(&bay);
13122 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13123 }
13124
13125 #[test]
13134 fn a_full_panel_that_found_nothing_is_clean() {
13135 assert!(round_is_clean(
13136 0,
13137 true,
13138 2,
13139 2,
13140 0,
13141 IncompleteReviewPolicy::Block
13142 ));
13143 }
13144
13145 #[test]
13146 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13147 assert!(!round_is_clean(
13148 0,
13149 true,
13150 1,
13151 2,
13152 0,
13153 IncompleteReviewPolicy::Block
13154 ));
13155 }
13156
13157 #[test]
13158 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13159 assert!(!round_is_clean(
13160 1,
13161 true,
13162 1,
13163 2,
13164 0,
13165 IncompleteReviewPolicy::Warn
13166 ));
13167 }
13168
13169 #[test]
13170 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13171 assert!(round_is_clean(
13172 0,
13173 true,
13174 1,
13175 2,
13176 0,
13177 IncompleteReviewPolicy::Warn
13178 ));
13179 }
13180
13181 #[test]
13182 fn a_full_panel_with_an_open_finding_is_not_clean() {
13183 assert!(!round_is_clean(
13184 1,
13185 true,
13186 2,
13187 2,
13188 0,
13189 IncompleteReviewPolicy::Block
13190 ));
13191 }
13192
13193 #[test]
13194 fn a_full_panel_with_a_red_e2e_is_not_clean() {
13195 assert!(!round_is_clean(
13196 0,
13197 false,
13198 2,
13199 2,
13200 0,
13201 IncompleteReviewPolicy::Block
13202 ));
13203 }
13204
13205 #[test]
13212 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13213 assert!(round_is_clean(
13216 0,
13217 true,
13218 1,
13219 2,
13220 1,
13221 IncompleteReviewPolicy::Block
13222 ));
13223 }
13224
13225 #[test]
13226 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13227 assert!(!round_is_clean(
13230 0,
13231 true,
13232 1,
13233 2,
13234 0,
13235 IncompleteReviewPolicy::Block
13236 ));
13237 }
13238
13239 #[test]
13240 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13241 assert!(!round_is_clean(
13242 1,
13243 true,
13244 1,
13245 2,
13246 1,
13247 IncompleteReviewPolicy::Block
13248 ));
13249 assert!(!round_is_clean(
13250 0,
13251 false,
13252 1,
13253 2,
13254 1,
13255 IncompleteReviewPolicy::Block
13256 ));
13257 }
13258
13259 #[test]
13260 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13261 assert!(!round_is_clean(
13265 0,
13266 true,
13267 0,
13268 2,
13269 2,
13270 IncompleteReviewPolicy::Block
13271 ));
13272 }
13273
13274 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13275 CommandOutcome {
13276 command: "test".to_owned(),
13277 code,
13278 output_tail: String::new(),
13279 duration_ms: 0,
13280 resource_blocked,
13281 }
13282 }
13283
13284 #[test]
13285 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13286 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13287 assert!(
13288 !verify_inconclusive(&[outcome(Some(1), false)]),
13289 "an ordinary failure is still evidence about the patch"
13290 );
13291 assert!(verify_inconclusive(&[outcome(None, true)]));
13292 assert!(
13293 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13294 "one inconclusive outcome taints the whole batch"
13295 );
13296 assert!(!verify_inconclusive(&[]));
13297 }
13298
13299 #[tokio::test]
13300 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13301 let calls = std::sync::atomic::AtomicUsize::new(0);
13305 let started = Instant::now();
13306 wait_for_pids_with(
13307 &[123],
13308 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13309 Duration::from_millis(5),
13310 Duration::from_secs(5),
13311 )
13312 .await;
13313 assert!(
13314 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13315 "must keep checking rather than deciding on the first answer"
13316 );
13317 assert!(
13318 started.elapsed() < Duration::from_secs(1),
13319 "must return the moment it is confirmed dead, not wait out the ceiling"
13320 );
13321 }
13322
13323 #[tokio::test]
13324 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13325 let started = Instant::now();
13326 wait_for_pids_with(
13327 &[123],
13328 |_| true, Duration::from_millis(5),
13330 Duration::from_millis(30),
13331 )
13332 .await;
13333 let elapsed = started.elapsed();
13334 assert!(
13335 elapsed >= Duration::from_millis(30),
13336 "must not give up before its own ceiling: {elapsed:?}"
13337 );
13338 assert!(
13339 elapsed < Duration::from_secs(1),
13340 "must not wait past its own ceiling either: {elapsed:?}"
13341 );
13342 }
13343
13344 #[tokio::test]
13345 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13346 let started = Instant::now();
13347 wait_for_pids_with(
13348 &[],
13349 |_| true,
13350 Duration::from_secs(5),
13351 Duration::from_secs(5),
13352 )
13353 .await;
13354 assert!(
13355 started.elapsed() < Duration::from_millis(200),
13356 "an empty pid list has nothing to confirm"
13357 );
13358 }
13359
13360 fn review_round(
13366 clean: bool,
13367 blocking: usize,
13368 answered: usize,
13369 expected: usize,
13370 progressed: bool,
13371 e2e_ok: bool,
13372 ) -> ReviewRound {
13373 ReviewRound {
13374 round: 1,
13375 head: "h".to_owned(),
13376 verified_head: None,
13377 verified_at: None,
13378 reviews: Vec::new(),
13379 e2e: vec![CommandOutcome {
13380 command: "test".to_owned(),
13381 code: Some(if e2e_ok { 0 } else { 1 }),
13382 output_tail: String::new(),
13383 duration_ms: 0,
13384 resource_blocked: false,
13385 }],
13386 verify_retried: false,
13387 e2e_deferred: false,
13388 e2e_defer_reason: None,
13389 fix: None,
13390 blocking,
13391 answered,
13392 expected,
13393 clean,
13394 progressed,
13395 vote_split: false,
13396 reconsideration: Vec::new(),
13397 verdict: None,
13398 }
13399 }
13400
13401 #[test]
13402 fn review_conclusion_is_none_when_nothing_has_run() {
13403 assert_eq!(review_conclusion(&[], 3), None);
13404 }
13405
13406 #[test]
13407 fn review_conclusion_is_none_while_rounds_remain() {
13408 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13409 assert_eq!(review_conclusion(&rounds, 3), None);
13410 }
13411
13412 #[test]
13413 fn review_conclusion_is_gating_once_a_round_is_clean() {
13414 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13415 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13416 }
13417
13418 #[test]
13419 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13420 let rounds = vec![
13421 review_round(false, 1, 2, 2, true, true),
13422 review_round(false, 1, 2, 2, true, true),
13423 ];
13424 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13425 }
13426
13427 #[test]
13428 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13429 let rounds = vec![
13430 review_round(false, 1, 2, 2, true, true),
13431 review_round(false, 1, 2, 2, true, false),
13432 ];
13433 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13434 }
13435
13436 #[test]
13437 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13438 let mut blocked = review_round(false, 1, 2, 2, true, false);
13445 blocked.e2e[0].resource_blocked = true;
13446 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13447 assert_eq!(review_conclusion(&rounds, 2), None);
13448 }
13449
13450 #[test]
13451 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13452 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13454 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13455 }
13456
13457 #[test]
13458 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13459 let rounds = vec![
13460 review_round(false, 1, 2, 2, false, true),
13461 review_round(false, 1, 2, 2, false, true),
13462 ];
13463 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13464 }
13465
13466 fn secs(n: u64) -> Duration {
13467 Duration::from_secs(n)
13468 }
13469
13470 #[tokio::test]
13473 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13474 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13475 if std::env::var_os(CHILD).is_none() {
13476 let output = std::process::Command::new(std::env::current_exe().unwrap())
13480 .args([
13481 "--exact",
13482 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13483 "--nocapture",
13484 ])
13485 .env(CHILD, "1")
13486 .env_remove("MAGI_WORKER_HOME")
13487 .env_remove("MAGI_WORKER_ID")
13488 .output()
13489 .unwrap();
13490 assert!(
13491 output.status.success()
13492 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13493 "{}\n{}",
13494 String::from_utf8_lossy(&output.stdout),
13495 String::from_utf8_lossy(&output.stderr)
13496 );
13497 return;
13498 }
13499 ask_test_home();
13500 let tmp = tempfile::tempdir().unwrap();
13501 let repo = tmp.path().join("repo");
13502 std::fs::create_dir_all(&repo).unwrap();
13503 init_repo(&repo);
13504 let g = |args: &[&str]| {
13505 let out = std::process::Command::new("git")
13506 .args(args)
13507 .current_dir(&repo)
13508 .quiet()
13509 .output()
13510 .unwrap();
13511 assert!(
13512 out.status.success(),
13513 "{}",
13514 String::from_utf8_lossy(&out.stderr)
13515 );
13516 };
13517 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13518 g(&["checkout", "-b", "historical"]);
13519 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13520 g(&["add", "."]);
13521 g(&["commit", "-m", "old alternative"]);
13522 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13523 g(&["checkout", "-b", "magi/task-selected", &base]);
13524 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13525 g(&["add", "."]);
13526 g(&["commit", "-m", "measurement infrastructure"]);
13527 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13528 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13529 g(&["add", "."]);
13530 g(&["commit", "-m", "dependent implementation"]);
13531 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13532 g(&["checkout", "main"]);
13533 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13534 g(&["add", "."]);
13535 g(&["commit", "-m", "unrelated base change"]);
13536 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13537 let origin = tmp.path().join("origin.git");
13538 g(&[
13539 "clone",
13540 "--bare",
13541 repo.to_str().unwrap(),
13542 origin.to_str().unwrap(),
13543 ]);
13544 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13545 g(&["fetch", "origin"]);
13546 let selection = crate::pr_recovery::Selection {
13547 url: "https://example.test/project/pull/1".into(),
13548 branch: "magi/task-selected".into(),
13549 head: head.clone(),
13550 source_head: None,
13551 source_run: None,
13552 source_proof: None,
13553 disposition: Some(crate::pr_recovery::Disposition::Review),
13554 instruction: "preserve measured behavior".into(),
13555 approved_at: jiff::Timestamp::now(),
13556 };
13557 let mut template = runner_at(RunStatus::Reviewing);
13558 template.state.repo = repo.clone();
13559 template.state.base_commit = current.clone();
13560 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13561 template.state.origin = Some(crate::run::Origin::queue("selected"));
13562 template.state.pr_head = Some(selection.branch.clone());
13563 template.state.pr_recovery = Some(selection.clone());
13564 let mut runner =
13565 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13566 .await
13567 .unwrap();
13568 let wt = runner.state.winner().unwrap().worktree.clone();
13569 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13570 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13571 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13572 assert!(runner.state.seeds.is_empty());
13573 assert_eq!(runner.state.instruction, "preserve measured behavior");
13574 runner.sync_to_base().await.unwrap();
13575 assert_eq!(
13576 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13577 "prerequisite"
13578 );
13579 assert_eq!(
13580 std::fs::read_to_string(wt.join("implementation")).unwrap(),
13581 "measured implementation"
13582 );
13583 assert_eq!(
13584 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13585 "current main change"
13586 );
13587 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13588 assert_eq!(
13589 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13590 head
13591 );
13592
13593 let mut partial = runner_at(RunStatus::Reviewing);
13595 partial.state.repo = repo.clone();
13596 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13597 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13598 run: Some("exit 1".into()),
13599 ..Default::default()
13600 }];
13601 let selection = runner.state.pr_recovery.clone().unwrap();
13602 partial.state.pr_recovery = Some(selection.clone());
13603 partial.state.pr_head = Some(selection.branch.clone());
13604 partial.state.origin = Some(crate::run::Origin::queue("selected"));
13605 let id = partial.state.id.clone();
13606 let error = match Runner::open_selected(
13607 &repo,
13608 partial.state,
13609 partial.roles,
13610 1,
13611 runner.state.base_commit.clone(),
13612 selection,
13613 )
13614 .await
13615 {
13616 Ok(_) => panic!("setup must fail"),
13617 Err(e) => e,
13618 };
13619 assert_eq!(
13620 error
13621 .downcast_ref::<crate::pr_recovery::StartupFailed>()
13622 .unwrap()
13623 .run,
13624 id
13625 );
13626 let failed = RunState::load(&id).unwrap();
13627 assert_eq!(failed.status, RunStatus::Blocked);
13628 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13629 assert!(Runner::resume(&id).is_err());
13630 runner.state.status = RunStatus::Ready;
13631 runner.state.driver_exited = true;
13632 runner.state.save().unwrap();
13633 let mut retry = runner_at(RunStatus::Reviewing);
13634 retry.state.repo = repo.clone();
13635 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13636 let mut reselected = runner.state.pr_recovery.clone().unwrap();
13637 reselected.approved_at = jiff::Timestamp::now();
13638 retry.state.origin = Some(crate::run::Origin::queue("selected"));
13639 retry.state.pr_head = Some(reselected.branch.clone());
13640 retry.state.pr_recovery = Some(reselected.clone());
13641 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13642 .await
13643 .unwrap();
13644 let restarted = Runner::open_selected(
13645 &repo,
13646 retry.state,
13647 retry.roles,
13648 1,
13649 runner.state.base_commit.clone(),
13650 reselected,
13651 )
13652 .await
13653 .unwrap();
13654 assert_eq!(
13655 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13656 .await
13657 .unwrap(),
13658 head
13659 );
13660 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13661 assert_eq!(
13662 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13663 head
13664 );
13665 }
13666
13667 #[tokio::test]
13668 #[cfg(unix)]
13669 async fn fallback_and_interrupted_restoration_require_coordination() {
13670 use std::os::fd::AsRawFd as _;
13671 ask_test_home();
13672 let tmp = tempfile::tempdir().unwrap();
13673 let mut runner = registered_runner(tmp.path()).await;
13674 runner.state.config.graph.timeout_resource_wait = 1;
13675 let winner = runner.state.candidates[0].clone();
13676 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13677 let leftover = winner.worktree.join("leftover");
13678 std::fs::write(&leftover, "untrusted edits").unwrap();
13679 let next = tmp.path().join("next-fixer");
13680 let mut failed = spec("first");
13681 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13682 let mut successor = spec("second");
13683 successor.command = vec![
13684 "sh".into(),
13685 "-c".into(),
13686 format!("touch '{}'; echo done", next.display()),
13687 ];
13688 let attempts = vec![
13689 (failed.clone(), "first".into()),
13690 (successor.clone(), "second".into()),
13691 ];
13692 let lock_path = runner.state.repo.join(".git/updater.lock");
13693 let external = std::fs::File::create(&lock_path).unwrap();
13694 assert_eq!(
13695 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13696 0
13697 );
13698 let artifacts = tmp.path().join("artifacts");
13699 let build = |spec, seat| SeatJob {
13700 spec,
13701 seat,
13702 cwd: winner.worktree.clone(),
13703 prompt: "fixture".into(),
13704 timeout: Duration::from_secs(10),
13705 allow_write: true,
13706 sessions: false,
13707 artifacts: artifacts.clone(),
13708 stem: "fix".into(),
13709 handover: None,
13710 };
13711 let (_, _, out) = runner
13712 .ask_fixer_among(
13713 attempts.clone(),
13714 "graft",
13715 None,
13716 Some((&winner.worktree, &head)),
13717 build,
13718 )
13719 .await;
13720 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13721 assert!(leftover.exists(), "contention must prevent clean");
13722 assert!(!next.exists());
13723 assert!(runner.state.handovers.is_empty());
13724 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13726 from_run: "earlier".into(),
13727 generation: 1,
13728 mode: crate::recovery::Mode::VerifyRepair,
13729 reasons: Vec::new(),
13730 reviewed_head: None,
13731 rounds: Vec::new(),
13732 open_findings: Vec::new(),
13733 evidence: None,
13734 criteria_inherited: true,
13735 repair: crate::recovery::Repair::Started {
13736 before: head.clone(),
13737 },
13738 });
13739 assert!(
13740 runner
13741 .recovery_repair(&winner)
13742 .await
13743 .unwrap_err()
13744 .is::<crate::repo_lock::ResourceBlocked>()
13745 );
13746 assert!(matches!(
13747 runner.state.recovery.as_ref().unwrap().repair,
13748 crate::recovery::Repair::Started { .. }
13749 ));
13750 assert!(leftover.exists());
13751 drop(external);
13752 runner.recovery_repair(&winner).await.unwrap();
13753 assert!(!leftover.exists());
13754 assert!(matches!(
13755 runner.state.recovery.as_ref().unwrap().repair,
13756 crate::recovery::Repair::Done { .. }
13757 ));
13758 std::fs::remove_file(&lock_path).unwrap();
13760 std::fs::create_dir(&lock_path).unwrap();
13761 std::fs::write(&leftover, "untrusted edits").unwrap();
13762 let (_, _, out) = runner
13763 .ask_fixer_among(
13764 attempts,
13765 "graft",
13766 None,
13767 Some((&winner.worktree, &head)),
13768 build,
13769 )
13770 .await;
13771 assert!(matches!(out, AgentOutcome::Failed(_)));
13772 assert!(leftover.exists());
13773 assert!(!next.exists());
13774 assert_eq!(
13775 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13776 head
13777 );
13778 }
13779
13780 #[cfg(unix)]
13781 async fn registered_runner(root: &Path) -> Runner {
13782 repository_runner(root, crate::config::test_coordination()).await
13783 }
13784
13785 async fn repository_runner(
13786 root: &Path,
13787 coordination: Option<crate::config::Coordination>,
13788 ) -> Runner {
13789 ask_test_home();
13790 let repo = root.join("repo");
13791 std::fs::create_dir_all(&repo).unwrap();
13792 init_repo(&repo);
13793 if coordination.is_some() {
13794 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13795 }
13796 let mut runner = runner_at(RunStatus::Prep);
13797 runner.state.repo = repo.clone();
13798 runner.state.config.coordination = coordination;
13799 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13800 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13801 runner.state.config.graph.advise = false;
13802 runner.roles.implementers = vec![spec("implement")];
13803 runner.roles.judges = vec![spec("judge")];
13804 runner.prep().await.unwrap();
13805 runner
13806 }
13807
13808 const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
13809
13810 #[tokio::test]
13811 async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
13812 let tmp = tempfile::tempdir().unwrap();
13813 let mut runner = repository_runner(tmp.path(), None).await;
13814 let prompts = runner.state.config.prompts.clone();
13815 let run = runner.state.id.clone();
13816 let ctx = WaveCtx {
13817 carry_seats: false,
13818 pauses: None,
13819 run: &run,
13820 node: "advise",
13821 prompts: &prompts,
13822 cache: None,
13823 round: None,
13824 };
13825 let jobs: Vec<_> = ["alpha", "beta"]
13826 .into_iter()
13827 .map(|id| SeatJob {
13828 spec: scripted(id, AUTH_OUTAGE),
13829 seat: SeatState::new(id, id, 7),
13830 prompt: "propose".into(),
13831 cwd: runner.state.candidates[0].worktree.clone(),
13832 timeout: Duration::from_secs(10),
13833 allow_write: false,
13834 sessions: false,
13835 artifacts: agent::artifacts_dir(&runner.state.dir()),
13836 stem: id.into(),
13837 handover: None,
13838 })
13839 .collect();
13840 let mut losses = Vec::new();
13841 let results = ask_wave_with(
13842 jobs.clone(),
13843 runner.sem.clone(),
13844 2,
13845 &[],
13846 &ctx,
13847 &mut losses,
13848 &mut runner.state,
13849 &|_: &str| Ok(()),
13850 )
13851 .await;
13852 assert!(
13853 results
13854 .iter()
13855 .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
13856 );
13857 assert_eq!(runner.state.quota.len(), 2);
13858 assert!(
13859 runner
13860 .state
13861 .quota
13862 .iter()
13863 .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
13864 );
13865 assert!(!runner.state.advise_attempted);
13866 runner.stop_for_resource().unwrap();
13867 assert_eq!(runner.state.status, RunStatus::Blocked);
13868 assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
13869 assert_eq!(
13870 runner.state.resource_block.as_ref().unwrap().kind,
13871 "agent authentication"
13872 );
13873 runner.state.resource_block = None;
13876 runner.state.quota.clear();
13877 let mut mixed = jobs;
13878 mixed[1].spec.command = vec!["false".into()];
13879 let results = ask_wave_with(
13880 mixed,
13881 runner.sem.clone(),
13882 0,
13883 &[],
13884 &ctx,
13885 &mut losses,
13886 &mut runner.state,
13887 &|_: &str| Ok(()),
13888 )
13889 .await;
13890 assert!(results.iter().all(|(_, res, _)| res.is_err()));
13891 assert!(runner.state.resource_block.is_none());
13892 assert_eq!(losses.len(), 1);
13893 }
13894
13895 #[tokio::test]
13896 async fn unauthenticated_implementation_keeps_its_candidate_pending() {
13897 let tmp = tempfile::tempdir().unwrap();
13898 let mut runner = repository_runner(tmp.path(), None).await;
13899 let implementer = scripted("implement", AUTH_OUTAGE);
13900 runner.roles.implementers = vec![implementer.clone()];
13901 runner.state.config.agents = vec![implementer];
13902 runner.implement().await.unwrap();
13903 assert_eq!(runner.state.status, RunStatus::Blocked);
13904 assert_eq!(runner.state.quota.len(), 1);
13905 assert_eq!(
13906 runner.state.quota[0].cause,
13907 agent::QuotaCause::Authentication
13908 );
13909 let candidate = &runner.state.candidates[0];
13910 assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
13911 assert_eq!(runner.state.seats["impl-A"].turns, 1);
13912 }
13913
13914 #[tokio::test]
13915 #[cfg(unix)]
13916 async fn parked_implementer_persists_completed_work() {
13917 for registered in [false, true] {
13918 let tmp = tempfile::tempdir().unwrap();
13919 let mut runner = registered_runner(tmp.path()).await;
13920 if !registered {
13921 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13922 }
13923 let marker = tmp.path().join("started");
13924 let release = tmp.path().join("release");
13925 let mut implementer = spec("implement");
13926 implementer.command = vec![
13927 "sh".into(),
13928 "-c".into(),
13929 format!(
13930 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13931 marker.display(),
13932 release.display()
13933 ),
13934 ];
13935 runner.roles.implementers = vec![implementer.clone()];
13936 runner.state.config.agents = vec![implementer];
13937 let pause = runner.pause.clone();
13938 let (result, ()) = tokio::join!(runner.implement(), async {
13939 tokio::time::timeout(Duration::from_secs(10), async {
13940 while !marker.exists() {
13941 tokio::time::sleep(Duration::from_millis(10)).await;
13942 }
13943 })
13944 .await
13945 .unwrap();
13946 pause.park();
13947 std::fs::write(&release, "go").unwrap();
13948 });
13949 result.unwrap();
13950 let candidate = &runner.state.candidates[0];
13951 assert!(candidate.commits > 0, "{candidate:?}");
13952 assert!(candidate.summary.contains("saved-result"));
13953 assert_eq!(candidate.files, 1);
13954 let before = candidate.commits;
13955 runner.implement().await.unwrap();
13956 assert_eq!(runner.state.candidates[0].commits, before);
13957 }
13958 }
13959
13960 #[tokio::test]
13961 #[cfg(unix)]
13962 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13963 let tmp = tempfile::tempdir().unwrap();
13964 let mut runner = registered_runner(tmp.path()).await;
13965 let winner = runner.state.candidates[0].clone();
13966 let cache = tmp.path().join("cache");
13967 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13968 runner.state.config.graph.timeout_resource_wait = 0;
13969 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13970 let crate::cache::AcquireOutcome::Acquired(held) =
13971 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13972 else {
13973 panic!("cache must be free")
13974 };
13975 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13976 from_run: "earlier".into(),
13977 generation: 1,
13978 mode: crate::recovery::Mode::VerifyRepair,
13979 reasons: Vec::new(),
13980 reviewed_head: None,
13981 rounds: Vec::new(),
13982 open_findings: Vec::new(),
13983 evidence: None,
13984 criteria_inherited: true,
13985 repair: crate::recovery::Repair::NotStarted,
13986 });
13987 runner.recovery_repair(&winner).await.unwrap();
13988 assert!(matches!(
13989 runner.state.recovery.as_ref().unwrap().repair,
13990 crate::recovery::Repair::WaitingForRestore { .. }
13991 ));
13992 drop(held);
13993 runner.state.resource_block = None;
13994 runner.state.config.graph.timeout_resource_wait = 1;
13995 runner.recovery_repair(&winner).await.unwrap();
13996 assert!(matches!(
13997 runner.state.recovery.as_ref().unwrap().repair,
13998 crate::recovery::Repair::Done { .. }
13999 ));
14000 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14001 }
14002
14003 #[tokio::test]
14004 #[cfg(unix)]
14005 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14006 use std::os::fd::AsRawFd as _;
14007 for graft in [false, true] {
14008 let tmp = tempfile::tempdir().unwrap();
14009 let mut runner = registered_runner(tmp.path()).await;
14010 runner.state.config.graph.timeout_resource_wait = 1;
14011 let winner = runner.state.candidates[0].clone();
14012 let marker = tmp.path().join("fixer-started");
14013 let mut fixer = spec("fixer");
14014 fixer.command = vec![
14015 "sh".into(),
14016 "-c".into(),
14017 format!("echo started >> '{}'; echo done", marker.display()),
14018 ];
14019 runner.roles.fixer = Some(vec![fixer]);
14020 if graft {
14021 let mut loser = winner.clone();
14022 loser.label = 'B';
14023 loser.index = 1;
14024 runner.state.candidates.push(loser);
14025 runner.state.config.graph.graft_runner_up = true;
14026 runner.state.tally = Some(Tally {
14027 first_choice: BTreeMap::new(),
14028 borda: BTreeMap::new(),
14029 winner: winner.label,
14030 rankings: 1,
14031 unanimous_initial: true,
14032 deliberated: false,
14033 changed_votes: 0,
14034 unanimous_final: true,
14035 tie_break: None,
14036 judges: 1,
14037 present: 1,
14038 quorum: 1,
14039 met_quorum: true,
14040 uncontested: None,
14041 });
14042 runner.state.judgements.push(crate::run::Judgement {
14043 judge: 1,
14044 seat: "judge-1".into(),
14045 agent: "judge".into(),
14046 ranking: vec![winner.label, 'B'],
14047 reasons: BTreeMap::new(),
14048 confidence: None,
14049 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14050 order: vec![0, 1],
14051 failed: None,
14052 duration_ms: 0,
14053 });
14054 } else {
14055 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14056 from_run: "earlier".into(),
14057 generation: 1,
14058 mode: crate::recovery::Mode::VerifyRepair,
14059 reasons: Vec::new(),
14060 reviewed_head: None,
14061 rounds: Vec::new(),
14062 open_findings: Vec::new(),
14063 evidence: None,
14064 criteria_inherited: true,
14065 repair: crate::recovery::Repair::NotStarted,
14066 });
14067 }
14068 let reader = crate::repo_lock::SeatLease::for_checkout(
14069 &winner.worktree,
14070 false,
14071 crate::config::test_coordination().as_ref(),
14072 )
14073 .await
14074 .unwrap();
14075 let pause = runner.pause.clone();
14076 let (result, ()) = tokio::join!(
14077 async {
14078 if graft {
14079 runner.graft().await
14080 } else {
14081 runner.recovery_repair(&winner).await
14082 }
14083 },
14084 async {
14085 tokio::time::sleep(Duration::from_millis(100)).await;
14086 pause.park();
14087 }
14088 );
14089 result.unwrap();
14090 assert!(!marker.exists(), "a blocked seat must not spawn");
14091 runner.state = RunState::load(&runner.state.id).unwrap();
14093 if graft {
14094 assert_eq!(
14095 runner.state.graft.as_ref().unwrap().outcome,
14096 GraftOutcome::WaitingForRestore
14097 );
14098 } else {
14099 assert!(matches!(
14100 runner.state.recovery.as_ref().unwrap().repair,
14101 crate::recovery::Repair::WaitingForRestore { .. }
14102 ));
14103 }
14104 drop(reader);
14105 runner.pause = Pause::new();
14106 runner.state.resource_block = None;
14107 let external =
14109 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14110 assert_eq!(
14111 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14112 0
14113 );
14114 let leftover = winner.worktree.join("leftover");
14115 std::fs::write(&leftover, "earlier fallback edits").unwrap();
14116 let result = if graft {
14117 runner.graft().await
14118 } else {
14119 runner.recovery_repair(&winner).await
14120 };
14121 assert!(
14122 result
14123 .unwrap_err()
14124 .is::<crate::repo_lock::ResourceBlocked>()
14125 );
14126 assert!(leftover.exists());
14127 assert!(!marker.exists());
14128 drop(external);
14129 runner.state.resource_block = None;
14130 let result = if graft {
14131 runner.graft().await
14132 } else {
14133 runner.recovery_repair(&winner).await
14134 };
14135 result.unwrap();
14136 assert!(!leftover.exists(), "restore must precede retry");
14137 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14138 if graft {
14139 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14140 } else {
14141 assert!(matches!(
14142 runner.state.recovery.as_ref().unwrap().repair,
14143 crate::recovery::Repair::Done { .. }
14144 ));
14145 }
14146 }
14147 }
14148
14149 #[tokio::test]
14150 #[cfg(unix)]
14151 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14152 use std::os::fd::AsRawFd as _;
14153 let tmp = tempfile::tempdir().unwrap();
14154 let mut runner = registered_runner(tmp.path()).await;
14155 runner.state.config.graph.timeout_resource_wait = 1;
14156 let winner = runner.state.candidates[0].clone();
14157 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14158 runner.state.tally = Some(Tally {
14159 first_choice: BTreeMap::new(),
14160 borda: BTreeMap::new(),
14161 winner: winner.label,
14162 rankings: 0,
14163 unanimous_initial: false,
14164 deliberated: false,
14165 changed_votes: 0,
14166 unanimous_final: false,
14167 tie_break: None,
14168 judges: 0,
14169 present: 0,
14170 quorum: 0,
14171 met_quorum: true,
14172 uncontested: Some("fixture".into()),
14173 });
14174 runner.state.graft = Some(GraftRecord {
14175 pre: pre.clone(),
14176 post: None,
14177 lines: None,
14178 limit: None,
14179 agent: None,
14180 outcome: GraftOutcome::Running,
14181 requested: Vec::new(),
14182 adoption: None,
14183 adoption_error: None,
14184 });
14185 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14186 git::commit_all(&winner.worktree, "unfinished graft")
14187 .await
14188 .unwrap();
14189 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14190 let leftover = winner.worktree.join("leftover");
14191 std::fs::write(&leftover, "untrusted edits").unwrap();
14192 let path = runner.state.repo.join(".git/updater.lock");
14193 let external = std::fs::File::create(&path).unwrap();
14194 assert_eq!(
14195 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14196 0
14197 );
14198 assert!(
14199 runner
14200 .graft()
14201 .await
14202 .unwrap_err()
14203 .is::<crate::repo_lock::ResourceBlocked>()
14204 );
14205 assert!(matches!(
14206 runner.state.graft.as_ref().unwrap().outcome,
14207 GraftOutcome::Running
14208 ));
14209 assert_eq!(
14210 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14211 unfinished
14212 );
14213 assert!(leftover.exists());
14214 drop(external);
14215 std::fs::remove_file(&path).unwrap();
14216 std::fs::create_dir(&path).unwrap();
14217 assert!(runner.graft().await.is_err());
14218 assert!(matches!(
14219 runner.state.graft.as_ref().unwrap().outcome,
14220 GraftOutcome::Running
14221 ));
14222 assert_eq!(
14223 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14224 unfinished
14225 );
14226 assert!(leftover.exists());
14227 std::fs::remove_dir(&path).unwrap();
14228 runner.graft().await.unwrap();
14229 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14230 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
14231 assert!(!leftover.exists());
14232 }
14233
14234 #[tokio::test]
14235 #[cfg(unix)]
14236 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
14237 use std::os::fd::AsRawFd as _;
14238 let tmp = tempfile::tempdir().unwrap();
14239 let mut runner = registered_runner(tmp.path()).await;
14240 runner.state.prep_complete = Some(false);
14241 runner.state.config.graph.timeout_resource_wait = 1;
14242 runner.state.save().unwrap();
14243 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14244 assert_eq!(
14245 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14246 0
14247 );
14248 runner.execute().await.unwrap();
14249 let disk = RunState::load(&runner.state.id).unwrap();
14250 assert!(disk.driver_exited);
14251 assert_eq!(disk.driver_pid, None);
14252 assert_eq!(runner.state.status, RunStatus::Blocked);
14253 assert_eq!(
14254 runner.state.resource_block.as_ref().unwrap().kind,
14255 "repository coordination"
14256 );
14257 assert!(runner.state.prep_failure.is_none());
14258 assert!(runner.state.seats.is_empty());
14259 runner.pause.park();
14260 tokio::time::timeout(Duration::from_secs(10), runner.execute())
14263 .await
14264 .unwrap()
14265 .unwrap();
14266 assert!(runner.state.parked);
14267 assert!(runner.state.prep_failure.is_none());
14268 let disk = RunState::load(&runner.state.id).unwrap();
14269 assert!(disk.driver_exited);
14270 assert_eq!(disk.driver_pid, None);
14271 }
14272
14273 #[tokio::test]
14274 #[cfg(unix)]
14275 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14276 use std::os::fd::AsRawFd as _;
14277 let tmp = tempfile::tempdir().unwrap();
14278 let mut runner = registered_runner(tmp.path()).await;
14279 runner.state.prep_complete = Some(false);
14280 runner.state.config.graph.timeout_resource_wait = 30;
14281 runner.state.save().unwrap();
14282 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14283 assert_eq!(
14284 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14285 0
14286 );
14287 assert!(
14288 tokio::time::timeout(Duration::from_millis(100), runner.execute())
14289 .await
14290 .is_err()
14291 );
14292 let disk = RunState::load(&runner.state.id).unwrap();
14293 assert!(disk.driver_exited);
14294 assert_eq!(disk.driver_pid, None);
14295 assert!(disk.status.resumable());
14296 assert!(runner.state.driver_exited);
14297 assert_eq!(runner.state.driver_pid, None);
14298 runner.state.save().unwrap();
14299 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14300 }
14301
14302 #[tokio::test]
14303 #[cfg(unix)]
14304 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14305 let tmp = tempfile::tempdir().unwrap();
14306 let mut runner = registered_runner(tmp.path()).await;
14307 let winner = runner.state.candidates[0].clone();
14308 let cache = tmp.path().join("cache");
14309 let started = tmp.path().join("pre-gate-started");
14310 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14311 runner.state.config.verify.gate =
14312 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14313 let home = crate::run::home();
14314 let owner =
14315 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14316 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14317 crate::cache::AcquireOutcome::Acquired(g) => g,
14318 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14319 };
14320 let mut gate = Box::pin(runner.run_gate(&winner));
14321 assert!(
14322 tokio::time::timeout(Duration::from_millis(150), &mut gate)
14323 .await
14324 .is_err()
14325 );
14326 assert!(!started.exists(), "pre-gate must wait for the cache too");
14327 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14331 let editor = crate::repo_lock::SeatLease::checkout_with(
14332 &winner.worktree,
14333 true,
14334 crate::config::test_coordination().as_ref(),
14335 &wait,
14336 )
14337 .await
14338 .unwrap();
14339 drop(editor);
14340 guard.release();
14341 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14342 .await
14343 .unwrap()
14344 .unwrap();
14345 drop(gate);
14346 assert!(outcomes.iter().all(CommandOutcome::ok));
14347 assert!(started.exists());
14348 }
14349
14350 #[tokio::test]
14351 #[cfg(unix)]
14352 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14353 use std::os::fd::AsRawFd as _;
14354
14355 ask_test_home();
14356 let tmp = tempfile::tempdir().unwrap();
14357 let repo = tmp.path().join("repo");
14358 std::fs::create_dir_all(&repo).unwrap();
14359 init_repo(&repo);
14360 let common = repo.join(".git");
14361 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14362 let mut runner = runner_at(RunStatus::Prep);
14363 runner.state.repo = repo.clone();
14364 runner.state.config.coordination = crate::config::test_coordination();
14365 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14366 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14367 runner.state.config.graph.advise = false;
14368 runner.roles.implementers = vec![spec("implement")];
14369 runner.roles.judges = vec![spec("judge")];
14370 runner.prep().await.unwrap();
14371 let winner = runner.state.candidates[0].clone();
14372 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14375 let completed = tmp.path().join("seat-completed");
14376 runner.roles.implementers[0].command = vec![
14377 "sh".into(),
14378 "-c".into(),
14379 format!("touch '{}'; echo completed", completed.display()),
14380 ];
14381 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14382 assert_eq!(
14383 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14384 0
14385 );
14386 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14387 let mut implement = Box::pin(runner.implement());
14388 tokio::select! {
14389 result = &mut implement => panic!("rescue must wait: {result:?}"),
14390 result = tokio::time::timeout(Duration::from_secs(10), async {
14391 while !completed.exists() {
14392 tokio::time::sleep(Duration::from_millis(20)).await;
14393 }
14394 }) => result.unwrap(),
14395 }
14396 assert!(
14397 tokio::time::timeout(Duration::from_millis(200), &mut implement)
14398 .await
14399 .is_err()
14400 );
14401 assert_eq!(
14402 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14403 before
14404 );
14405 assert_eq!(
14406 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14407 0
14408 );
14409 tokio::time::timeout(Duration::from_secs(10), &mut implement)
14410 .await
14411 .unwrap()
14412 .unwrap();
14413 drop(implement);
14414 assert_ne!(
14415 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14416 before
14417 );
14418 assert!(git::is_clean(&winner.worktree).await.unwrap());
14419
14420 let started = tmp.path().join("pre-gate-started");
14421 let proceed = tmp.path().join("pre-gate-proceed");
14422 let checked = tmp.path().join("gate-ran");
14423 runner.state.config.verify.pre_gate = vec![format!(
14424 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14425 started.display(),
14426 proceed.display()
14427 )];
14428 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14429 assert_eq!(
14430 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14431 0
14432 );
14433 let mut gate = Box::pin(runner.run_gate(&winner));
14434 assert!(
14435 tokio::time::timeout(Duration::from_millis(200), &mut gate)
14436 .await
14437 .is_err()
14438 );
14439 assert!(!started.exists());
14440 assert!(!checked.exists());
14441 assert!(git::is_clean(&winner.worktree).await.unwrap());
14442 assert_eq!(
14443 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14444 0
14445 );
14446 tokio::select! {
14447 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14448 result = tokio::time::timeout(Duration::from_secs(10), async {
14449 while !started.exists() {
14450 tokio::time::sleep(Duration::from_millis(20)).await;
14451 }
14452 }) => result.unwrap(),
14453 }
14454 assert_eq!(
14455 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14456 -1,
14457 "mechanical edits must hold the real lock"
14458 );
14459 std::fs::write(proceed, "ready").unwrap();
14460 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14461 .await
14462 .unwrap()
14463 .unwrap();
14464 drop(gate);
14465 assert!(outcomes.iter().all(CommandOutcome::ok));
14466 assert!(checked.exists());
14467 assert!(runner.state.pre_gate_commit.is_some());
14468 assert!(git::is_clean(&winner.worktree).await.unwrap());
14469 assert_eq!(
14470 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14471 0,
14472 "gate completion must release coordination"
14473 );
14474 drop(outsider);
14475 std::fs::remove_file(common.join("updater.lock")).unwrap();
14476 std::fs::create_dir(common.join("updater.lock")).unwrap();
14477 std::fs::remove_file(&started).unwrap();
14478 std::fs::remove_file(&checked).unwrap();
14479 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14480 let error = runner.run_gate(&winner).await.unwrap_err();
14481 assert!(
14482 error
14483 .to_string()
14484 .contains("acquire mandatory repository lock")
14485 );
14486 assert!(
14487 !started.exists(),
14488 "failed acquisition must prevent pre-gate"
14489 );
14490 assert!(!checked.exists(), "failed acquisition must prevent gate");
14491 assert!(git::is_clean(&winner.worktree).await.unwrap());
14492 assert_eq!(
14493 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14494 head
14495 );
14496 }
14497
14498 #[tokio::test]
14499 #[cfg(unix)]
14500 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14501 use std::os::fd::AsRawFd as _;
14502 use std::time::Duration;
14503
14504 ask_test_home();
14505 let tmp = tempfile::tempdir().unwrap();
14506 let repo = tmp.path().join("repo");
14507 std::fs::create_dir_all(&repo).unwrap();
14508 init_repo(&repo);
14509 let common = repo.join(".git");
14510 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14511 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14512 assert_eq!(
14513 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14514 0
14515 );
14516 let mut runner = runner_at(RunStatus::Prep);
14517 runner.state.repo = repo.clone();
14518 runner.state.config.coordination = crate::config::test_coordination();
14519 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14520 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14521 runner.state.config.graph.advise = true;
14522 runner.state.config.graph.advisors = 1;
14523 runner.roles.implementers = vec![spec("implement")];
14524 runner.roles.judges = vec![spec("judge")];
14525 let root = runner.state.worktree_root();
14526 let started = tmp.path().join("setup-started");
14527 let proceed = tmp.path().join("setup-proceed");
14528 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14529 run: Some(format!(
14530 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14531 started.display(),
14532 proceed.display()
14533 )),
14534 ..Default::default()
14535 }];
14536 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14537 let config = git::git(&repo, &["config", "--local", "--list"])
14538 .await
14539 .unwrap();
14540 let mut prep = Box::pin(runner.prep());
14541 assert!(
14542 tokio::time::timeout(Duration::from_millis(200), &mut prep)
14543 .await
14544 .is_err()
14545 );
14546 assert!(
14547 !root.exists(),
14548 "contention must prevent even checkout creation"
14549 );
14550 assert!(!started.exists(), "contention must prevent setup");
14551 assert_eq!(
14552 git::git(&repo, &["branch", "--list"]).await.unwrap(),
14553 branches
14554 );
14555 assert_eq!(
14556 git::git(&repo, &["config", "--local", "--list"])
14557 .await
14558 .unwrap(),
14559 config
14560 );
14561 drop(lock);
14562 tokio::select! {
14563 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14564 result = tokio::time::timeout(Duration::from_secs(10), async {
14565 while !started.exists() {
14566 tokio::time::sleep(Duration::from_millis(20)).await;
14567 }
14568 }) => result.unwrap(),
14569 }
14570 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14571 assert_eq!(
14572 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14573 -1
14574 );
14575 assert!(
14576 tokio::time::timeout(
14577 Duration::from_millis(100),
14578 crate::repo_lock::SeatLease::for_checkout(
14579 &repo,
14580 true,
14581 crate::config::test_coordination().as_ref()
14582 )
14583 )
14584 .await
14585 .is_err(),
14586 "preparation must also exclude local editors"
14587 );
14588 std::fs::write(proceed, "ready").unwrap();
14589 tokio::time::timeout(Duration::from_secs(10), &mut prep)
14590 .await
14591 .unwrap()
14592 .unwrap();
14593 drop(prep);
14594 assert_eq!(runner.state.prep_complete, Some(true));
14595 assert!(root.join("judge-1").exists());
14596 assert!(root.join("advisor-1").exists());
14597 assert_eq!(
14598 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14599 0
14600 );
14601 }
14602
14603 #[tokio::test]
14604 #[cfg(unix)]
14605 async fn preparation_lock_failure_prevents_repository_changes() {
14606 ask_test_home();
14607 let tmp = tempfile::tempdir().unwrap();
14608 let repo = tmp.path().join("repo");
14609 std::fs::create_dir_all(&repo).unwrap();
14610 init_repo(&repo);
14611 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14612 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14615 let mut runner = runner_at(RunStatus::Prep);
14616 runner.state.repo = repo.clone();
14617 runner.state.config.coordination = crate::config::test_coordination();
14618 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14619 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14620 runner.roles.implementers = vec![spec("implement")];
14621 let error = runner.prep().await.unwrap_err();
14622 assert!(
14623 error
14624 .to_string()
14625 .contains("acquire mandatory repository lock")
14626 );
14627 assert!(!runner.state.worktree_root().exists());
14628 assert!(runner.state.candidates.is_empty());
14629 assert!(!repo.join(".git/config.worktree").exists());
14630 assert_eq!(
14631 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14632 "* main"
14633 );
14634 }
14635
14636 #[tokio::test]
14637 async fn preparation_records_creation_intent_before_git_can_fail() {
14638 ask_test_home();
14639 let tmp = tempfile::tempdir().unwrap();
14640 let repo = tmp.path().join("repo");
14641 std::fs::create_dir_all(&repo).unwrap();
14642 init_repo(&repo);
14643 let mut runner = runner_at(RunStatus::Prep);
14644 runner.state.repo = repo.clone();
14645 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14646 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14647 runner.state.config.blind.commit_msg_hook = false;
14648 runner.roles.implementers = vec![spec("one")];
14649 let label = blind::assign_labels(1, runner.state.seed)[0];
14650 let branch = runner.state.branch_for(label);
14651 git::git(&repo, &["branch", &branch]).await.unwrap();
14654 assert!(runner.prep().await.is_err());
14655 let saved = RunState::load(&runner.state.id).unwrap();
14656 assert!(saved.candidates.is_empty());
14657 let event = saved
14658 .events
14659 .iter()
14660 .find(|e| e.node == "prep-intent")
14661 .unwrap();
14662 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14663 assert_eq!(intent["branch"], branch);
14664 assert_eq!(intent["start"], runner.state.base_commit);
14665 assert_eq!(
14666 intent["worktree"],
14667 runner
14668 .state
14669 .worktree_root()
14670 .join(format!("cand-{label}"))
14671 .to_string_lossy()
14672 .as_ref()
14673 );
14674 assert_eq!(intent["picks"], serde_json::json!([]));
14675 }
14676
14677 #[tokio::test]
14678 async fn invalid_pinned_sources_create_no_partial_worktrees() {
14679 ask_test_home();
14680 let tmp = tempfile::tempdir().unwrap();
14681 let repo = tmp.path().join("repo");
14682 std::fs::create_dir_all(&repo).unwrap();
14683 init_repo(&repo);
14684 let mut runner = runner_at(RunStatus::Prep);
14685 runner.state.repo = repo.clone();
14686 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14687 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14688 runner.state.sources.pick_commits.push(refs::Input {
14689 token: "missing".into(),
14690 sha: "0".repeat(40),
14691 provenance: refs::Provenance::ExplicitCli,
14692 });
14693 assert!(runner.prep().await.is_err());
14694 assert!(runner.state.candidates.is_empty());
14695 assert!(!runner.state.worktree_root().exists());
14696 }
14697
14698 #[tokio::test]
14699 async fn prose_and_legacy_inference_never_import_into_candidates() {
14700 ask_test_home();
14701 let tmp = tempfile::tempdir().unwrap();
14702 let repo = tmp.path().join("repo");
14703 std::fs::create_dir_all(&repo).unwrap();
14704 init_repo(&repo);
14705 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14706 git::git(&repo, &["checkout", "-b", "magi/history/A"])
14707 .await
14708 .unwrap();
14709 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14710 git::git(&repo, &["add", "."]).await.unwrap();
14711 git::git(&repo, &["commit", "-m", "historical implementation"])
14712 .await
14713 .unwrap();
14714 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14715 git::git(&repo, &["checkout", "main"]).await.unwrap();
14716 for instruction in [
14717 format!("Example: magi/history/A at {source}"),
14718 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14719 format!("Never import magi/history/A or {source}"),
14720 format!("Conversation answer: I mentioned {source} only as background"),
14721 ] {
14722 let mut runner = runner_at(RunStatus::Prep);
14723 runner.state.repo = repo.clone();
14724 runner.state.base_commit = base.clone();
14725 runner.state.instruction = instruction.clone();
14726 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14727 runner.state.config.blind.commit_msg_hook = false;
14728 runner.state.seeds = vec![refs::Seed {
14729 token: "magi/history/A".into(),
14730 sha: source.clone(),
14731 kind: refs::SeedKind::Unmerged,
14732 branch: true,
14733 detail: String::new(),
14734 }];
14735 runner.roles.implementers = vec![spec("one")];
14736 assert_eq!(seeded_instruction(&runner.state), instruction);
14737 let mut legacy = serde_json::to_value(&runner.state).unwrap();
14739 legacy.as_object_mut().unwrap().remove("sources");
14740 runner.state = serde_json::from_value(legacy).unwrap();
14741 assert_eq!(runner.state.sources, refs::Sources::default());
14742 runner.state.save().unwrap();
14743 runner.state = RunState::load(&runner.state.id).unwrap();
14744 runner.prep().await.unwrap();
14745 for candidate in &runner.state.candidates {
14746 assert_eq!(
14747 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14748 base
14749 );
14750 assert!(!candidate.worktree.join("unwanted").exists());
14751 }
14752 }
14753 }
14754
14755 #[tokio::test]
14756 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14757 ask_test_home();
14758 let tmp = tempfile::tempdir().unwrap();
14759 let repo = tmp.path().join("repo");
14760 std::fs::create_dir_all(&repo).unwrap();
14761 init_repo(&repo);
14762 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14763 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14764 std::fs::write(repo.join("referenced"), "required commit").unwrap();
14765 git::git(&repo, &["add", "."]).await.unwrap();
14766 git::git(&repo, &["commit", "-m", "required seed"])
14767 .await
14768 .unwrap();
14769 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14770 git::git(&repo, &["checkout", "main"]).await.unwrap();
14771 let mut runner = runner_at(RunStatus::Prep);
14772 runner.state.repo = repo.clone();
14773 runner.state.base_commit = base.clone();
14774 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14775 runner.state.config.graph.advise = false;
14776 runner.state.config.blind.commit_msg_hook = false;
14777 runner.state.sources =
14778 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14779 .await
14780 .unwrap();
14781 runner.roles.implementers = vec![spec("one"), spec("two")];
14782 runner.roles.judges = vec![spec("judge")];
14783 let label = blind::assign_labels(2, runner.state.seed)[0];
14784 let branch = runner.state.branch_for(label);
14785 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14786 git::worktree_add_branch(&repo, &worktree, &branch, &base)
14787 .await
14788 .unwrap();
14789 runner.state.candidates.push(Candidate {
14790 index: 0,
14791 label,
14792 agent: "one".into(),
14793 branch,
14794 worktree,
14795 summary: String::new(),
14796 stat: String::new(),
14797 files: 0,
14798 commits: 0,
14799 empty: false,
14800 failed: None,
14801 verified_noop: None,
14802 duration_ms: 0,
14803 folded: false,
14804 });
14805 runner.state.save().unwrap();
14806 runner.state = RunState::load(&runner.state.id).unwrap();
14808 assert_eq!(runner.state.prep_complete, Some(false));
14809 runner.prep().await.unwrap();
14810 assert_eq!(runner.state.prep_complete, Some(true));
14811 assert_eq!(runner.state.candidates.len(), 2);
14812 for candidate in &runner.state.candidates {
14813 assert_eq!(
14814 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14815 git::tree_of(&repo, &seed).await.unwrap()
14816 );
14817 assert_eq!(
14818 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14819 "required commit"
14820 );
14821 }
14822 assert!(runner.state.worktree_root().join("judge-1").exists());
14823 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14824 runner.prep().await.unwrap();
14825 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14826
14827 runner.state.prep_complete = Some(false);
14828 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14829 run: Some(format!(
14830 "test -f '{}'",
14831 tmp.path().join("dependency-ready").display()
14832 )),
14833 ..Default::default()
14834 }];
14835 assert!(runner.prep().await.is_err());
14836 assert!(runner.state.prep_failure.is_some());
14837 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14838 runner.state = RunState::load(&runner.state.id).unwrap();
14839 assert!(
14840 runner.prep().await.is_err(),
14841 "correcting the cause alone must not auto-retry"
14842 );
14843 let failed_at = runner.state.events.last().unwrap().at;
14844 assert!(
14845 runner.retry_preparation(failed_at).is_err(),
14846 "old approvals cannot unlock the hold"
14847 );
14848 let mut task = crate::queue::Task::new(
14849 "retry".into(),
14850 "retry setup".into(),
14851 repo.clone(),
14852 crate::queue::Source::Human,
14853 );
14854 task.start(runner.state.id.clone());
14855 task.hold_machine(Some("setup failed".into()));
14856 task.release_operator("cli", None);
14857 runner
14858 .retry_preparation(task.resumption.as_ref().unwrap().at)
14859 .unwrap();
14860 runner.prep().await.unwrap();
14861 assert!(runner.state.prep_failure.is_none());
14862 assert_eq!(runner.state.prep_complete, Some(true));
14863 assert_eq!(runner.state.candidates.len(), 2);
14864 assert!(runner.state.worktree_root().join("judge-1").exists());
14865 for candidate in &runner.state.candidates {
14866 assert_eq!(
14867 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14868 git::tree_of(&repo, &seed).await.unwrap()
14869 );
14870 }
14871 }
14872
14873 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14874 let mut heads = Vec::new();
14875 for candidate in candidates {
14876 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14877 }
14878 heads
14879 }
14880
14881 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14882 ask_test_home();
14883 let tmp = tempfile::tempdir().unwrap();
14884 let repo = tmp.path().join("repo");
14885 std::fs::create_dir(&repo).unwrap();
14886 init_repo(&repo);
14887 let mut runner = runner_at(RunStatus::Reviewing);
14888 runner.state.repo = repo.clone();
14889 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14890 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14891 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14892 let branch = format!("candidate-{label}");
14893 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14894 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14895 .await
14896 .unwrap();
14897 if label == 'B' {
14898 std::fs::write(worktree.join("change"), "winner").unwrap();
14899 git::rescue_commit(&worktree, "candidate change")
14900 .await
14901 .unwrap();
14902 }
14903 runner.state.candidates.push(Candidate {
14904 index,
14905 label,
14906 agent: "alpha".into(),
14907 branch,
14908 worktree,
14909 summary: "completed reply".into(),
14910 stat: String::new(),
14911 files: usize::from(label == 'B'),
14912 commits: usize::from(label == 'B'),
14913 empty: label == 'A',
14914 failed: None,
14915 verified_noop: None,
14916 duration_ms: 1,
14917 folded: false,
14918 });
14919 }
14920 runner.state.judge_skipped = true;
14921 runner.state.tally = Some(Tally {
14922 first_choice: BTreeMap::new(),
14923 borda: BTreeMap::new(),
14924 winner: 'B',
14925 rankings: 0,
14926 unanimous_initial: false,
14927 deliberated: false,
14928 changed_votes: 0,
14929 unanimous_final: false,
14930 tie_break: None,
14931 judges: 0,
14932 present: 0,
14933 quorum: 0,
14934 met_quorum: true,
14935 uncontested: Some("only B produced a change".into()),
14936 });
14937 runner.state.save().unwrap();
14938 (tmp, runner)
14939 }
14940
14941 #[tokio::test]
14942 #[cfg(unix)]
14943 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14944 for (solo, failed_checkpoint) in
14945 [(true, false), (false, false), (true, true), (false, true)]
14946 {
14947 let (_tmp, mut runner) = preservation_resume_fixture().await;
14948 if solo {
14949 runner.state.candidates.retain(|c| c.label == 'A');
14950 }
14951 runner.state.candidates[0].empty = false;
14952 runner.state.candidates[0].summary.clear();
14953 runner.state.tally = None;
14954 runner.state.judge_skipped = false;
14955 runner.state.status = RunStatus::Implementing;
14956 runner.state.config.disk.min_free_bytes = u64::MAX;
14957 let mut implementer = spec("alpha");
14958 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14959 runner.roles.implementers = vec![implementer];
14960 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14961 if failed_checkpoint {
14962 std::fs::create_dir_all(&patch).unwrap();
14963 }
14964 runner.state.save().unwrap();
14965 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14966 runner.state = RunState::load(&runner.state.id).unwrap();
14967 assert_eq!(runner.state.status, RunStatus::Blocked);
14968 assert!(runner.state.interrupted_implementations.contains(&'A'));
14969 assert_eq!(runner.state.candidates[0].commits, 0);
14970 assert!(!runner.state.candidates[0].empty);
14971 assert!(runner.state.candidates[0].failed.is_none());
14972 if failed_checkpoint {
14973 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14974 std::fs::remove_dir(patch).unwrap();
14975 }
14976 runner.state.resource_block = None;
14977 runner.state.config.disk.min_free_bytes = 0;
14978 runner.roles.implementers[0].command = vec![
14979 "sh".into(),
14980 "-c".into(),
14981 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14982 ];
14983 runner.implement().await.unwrap();
14984 assert!(runner.state.interrupted_implementations.is_empty());
14985 assert!(runner.state.candidate_preservation.is_empty());
14986 assert_eq!(runner.state.candidates[0].commits, 1);
14987 assert!(!runner.state.candidates[0].empty);
14988 assert!(
14989 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14990 .unwrap()
14991 .contains("retried empty interrupted seat")
14992 );
14993 if !solo {
14994 assert_eq!(runner.state.candidates[1].commits, 1);
14995 }
14996 }
14997 }
14998
14999 #[tokio::test]
15000 #[cfg(unix)]
15001 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15002 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15003 let (_tmp, mut runner) = preservation_resume_fixture().await;
15004 runner.state.candidates.retain(|c| c.label == 'A');
15005 runner.state.candidates[0].empty = false;
15006 runner.state.candidates[0].summary.clear();
15007 runner.state.tally = None;
15008 runner.state.judge_skipped = false;
15009 runner.state.status = RunStatus::Implementing;
15010 runner.state.config.disk.min_free_bytes = u64::MAX;
15013 let wt = runner.state.candidates[0].worktree.clone();
15014 std::fs::write(
15015 wt.join("interrupted-source"),
15016 "retain implementation work\n",
15017 )
15018 .unwrap();
15019 if committed {
15020 git::rescue_commit(&wt, "implementation checkpoint")
15021 .await
15022 .unwrap();
15023 }
15024 let lock = git::git(
15025 &wt,
15026 &[
15027 "rev-parse",
15028 "--path-format=absolute",
15029 "--git-path",
15030 "index.lock",
15031 ],
15032 )
15033 .await
15034 .unwrap();
15035 if block_rescue {
15036 std::fs::write(&lock, "fixture-owned lock").unwrap();
15037 }
15038 let mut implementer = spec("alpha");
15039 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15040 runner.roles.implementers = vec![implementer];
15041 runner.state.save().unwrap();
15042 let result = runner.implement().await;
15043 assert_eq!(result.is_err(), block_rescue);
15044 assert_eq!(
15045 runner.state.resource_block.as_ref().unwrap().kind,
15046 "disk pressure"
15047 );
15048 assert!(
15049 runner
15050 .state
15051 .events
15052 .iter()
15053 .any(|e| e.message.contains("agent execution interrupted"))
15054 );
15055 assert!(
15056 runner
15057 .state
15058 .dir()
15059 .join("artifacts/candidate-A-recovery/staged.patch")
15060 .exists()
15061 );
15062 if block_rescue {
15063 let saved = RunState::load(&runner.state.id).unwrap();
15064 assert!(saved.candidate_preservation.contains_key(&'A'));
15065 assert!(!saved.candidates[0].empty);
15066 assert_eq!(
15067 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15068 "retain implementation work\n"
15069 );
15070 std::fs::remove_file(lock).unwrap();
15071 runner.state = saved;
15072 } else {
15073 assert_eq!(runner.state.status, RunStatus::Blocked);
15074 assert_eq!(runner.state.candidates[0].commits, 1);
15075 }
15076 runner.roles.implementers.clear();
15079 runner.state.config.disk.min_free_bytes = 0;
15080 runner.state.resource_block = None;
15081 runner.implement().await.unwrap();
15082 assert!(runner.state.candidate_preservation.is_empty());
15083 assert_eq!(runner.state.candidates[0].commits, 1);
15084 assert!(!runner.state.candidates[0].empty);
15085 assert!(git::is_clean(&wt).await.unwrap());
15086 assert!(
15087 crate::run::read_artifact(&runner.state, "cand-A.patch")
15088 .unwrap()
15089 .contains("retain implementation work")
15090 );
15091 }
15092 }
15093
15094 #[tokio::test]
15095 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15096 let (_tmp, mut runner) = preservation_resume_fixture().await;
15097 runner.implement().await.unwrap();
15099 assert!(runner.state.candidate_preservation.is_empty());
15100 assert!(
15101 !runner
15102 .state
15103 .dir()
15104 .join("artifacts/candidate-A-recovery")
15105 .exists()
15106 );
15107 runner.fold_losers().await.unwrap();
15108 assert!(!runner.state.candidates[0].worktree.exists());
15109 for schema in [14, crate::run::SCHEMA] {
15110 runner.state.schema = schema;
15111 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15112 runner.state.save().unwrap();
15113 runner.state = RunState::load(&runner.state.id).unwrap();
15114 runner.implement().await.unwrap();
15115 assert!(runner.state.candidate_preservation.is_empty());
15116 assert!(runner.state.candidates[0].folded);
15117 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15118 }
15119 }
15120
15121 #[tokio::test]
15122 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15123 let (_tmp, mut runner) = preservation_resume_fixture().await;
15124 runner.state.candidates.retain(|c| c.label == 'A');
15125 runner.state.tally = None;
15126 runner.state.judge_skipped = false;
15127 let candidate = &mut runner.state.candidates[0];
15128 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15129 candidate.failed = Some("deployment validation unavailable".into());
15130 std::fs::write(
15131 candidate.worktree.join("repair"),
15132 "future launcher policy\n",
15133 )
15134 .unwrap();
15135 runner
15136 .state
15137 .candidate_preservation
15138 .insert('A', "awaiting persistence".into());
15139 runner.persist_candidates().await.unwrap();
15140 let candidate = &runner.state.candidates[0];
15141 assert!(!candidate.empty);
15142 assert_eq!(candidate.commits, 1);
15143 assert_eq!(candidate.files, 1);
15144 assert!(candidate.failed.is_none());
15145 assert!(candidate.summary.contains("EPERM"));
15146 assert!(
15147 crate::run::read_artifact(&runner.state, "cand-A.patch")
15148 .unwrap()
15149 .contains("future launcher policy")
15150 );
15151 assert!(runner.state.candidate_preservation.is_empty());
15152 }
15153
15154 #[tokio::test]
15155 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15156 let (_tmp, mut runner) = preservation_resume_fixture().await;
15157 let wt = runner.state.candidates[0].worktree.clone();
15158 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15159 runner.state.schema = 14;
15160 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15161 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15162 let tally = serde_json::to_value(&runner.state.tally).unwrap();
15163 for _ in 0..2 {
15164 assert!(runner.implement().await.is_err());
15165 assert_eq!(runner.state.status, RunStatus::Blocked);
15166 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15167 assert!(
15168 runner.state.candidates[0].empty,
15169 "old verdict must remain unusable"
15170 );
15171 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15172 assert!(runner.state.judge_skipped);
15173 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15174 assert!(
15175 crate::run::read_artifact(&runner.state, "cand-A.patch")
15176 .unwrap()
15177 .contains("owned edits")
15178 );
15179 assert!(wt.exists());
15180 assert_eq!(
15181 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15182 .await
15183 .unwrap(),
15184 1
15185 );
15186 runner.state = RunState::load(&runner.state.id).unwrap();
15187 }
15188 }
15189
15190 #[tokio::test]
15191 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15192 let (_tmp, mut runner) = preservation_resume_fixture().await;
15193 let wt = runner.state.candidates[0].worktree.clone();
15194 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15195 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15196 let index_lock = PathBuf::from(
15197 git::git(
15198 &wt,
15199 &[
15200 "rev-parse",
15201 "--path-format=absolute",
15202 "--git-path",
15203 "index.lock",
15204 ],
15205 )
15206 .await
15207 .unwrap(),
15208 );
15209 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15210 runner.state.schema = 14;
15211 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15212 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15213 assert!(runner.implement().await.is_err());
15214 assert!(runner.after_implement().is_err());
15215 assert!(runner.fold_losers().await.is_err());
15216 assert_eq!(
15217 std::fs::read_to_string(wt.join("recovered")).unwrap(),
15218 "failed seat edits"
15219 );
15220 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15221 runner.state = RunState::load(&runner.state.id).unwrap();
15222 std::fs::remove_file(index_lock).unwrap();
15223 assert!(runner.implement().await.is_err());
15225 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15226 assert!(runner.state.candidates[0].empty);
15227 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15228 assert!(
15229 crate::run::read_artifact(&runner.state, "cand-A.patch")
15230 .unwrap()
15231 .contains("failed seat edits")
15232 );
15233 assert!(wt.exists());
15234 }
15235
15236 #[tokio::test]
15237 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
15238 let (_tmp, mut runner) = preservation_resume_fixture().await;
15239 let repo = runner.state.repo.clone();
15240 let common = git::git(
15241 &repo,
15242 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
15243 )
15244 .await
15245 .unwrap();
15246 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
15247 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
15248 assert!(runner.fold_losers().await.is_err());
15249 assert!(!runner.state.candidates[0].worktree.exists());
15250 assert!(!runner.state.candidates[0].folded);
15251 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
15252 runner.state = RunState::load(&runner.state.id).unwrap();
15253 std::fs::remove_file(branch_lock).unwrap();
15254 let judge = runner.state.worktree_root().join("judge-1");
15255 git::worktree_add_detached(&repo, &judge, "main")
15256 .await
15257 .unwrap();
15258 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
15259 runner.roles.judges = vec![spec("judge")];
15260 assert!(runner.fold_losers().await.is_err());
15261 runner.state = RunState::load(&runner.state.id).unwrap();
15262 assert!(
15263 runner.state.candidates[0].folded,
15264 "candidate progress saved before judge failure"
15265 );
15266 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15267 assert!(judge.join("uncommitted").exists());
15268 std::fs::remove_file(judge.join("uncommitted")).unwrap();
15269 runner.fold_losers().await.unwrap();
15270 assert!(!judge.exists());
15271 runner.fold_losers().await.unwrap();
15272 }
15273
15274 #[tokio::test]
15275 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15276 let (_tmp, mut runner) = preservation_resume_fixture().await;
15277 runner.state.candidates.retain(|c| c.label == 'B');
15278 runner.state.tally = None;
15279 runner.state.judge_skipped = false;
15280 runner.state.prep_complete = Some(true);
15281 runner.state.config.graph.advise = false;
15282 runner.state.config.graph.selected_implementer = Some("alpha".into());
15283 let wt = runner.state.candidates[0].worktree.clone();
15284 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15285 runner
15286 .state
15287 .candidate_preservation
15288 .insert('B', "awaiting persistence".into());
15289 let index_lock = git::git(
15290 &wt,
15291 &[
15292 "rev-parse",
15293 "--path-format=absolute",
15294 "--git-path",
15295 "index.lock",
15296 ],
15297 )
15298 .await
15299 .unwrap();
15300 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15301 runner.state.save().unwrap();
15302 assert!(runner.execute_graph().await.is_err());
15303 assert_eq!(runner.state.status, RunStatus::Blocked);
15304 assert!(runner.state.candidate_preservation.contains_key(&'B'));
15305 std::fs::remove_file(index_lock).unwrap();
15306 for _ in 0..2 {
15309 runner.state = RunState::load(&runner.state.id).unwrap();
15310 runner.pause.park();
15311 runner.execute_graph().await.unwrap();
15312 assert_eq!(runner.state.status, RunStatus::Implementing);
15313 assert!(runner.state.candidate_preservation.is_empty());
15314 assert_eq!(runner.state.candidates[0].commits, 2);
15315 assert!(runner.state.tally.is_none());
15316 }
15317 runner.implement().await.unwrap();
15318 assert_eq!(runner.state.status, RunStatus::Judging);
15319 runner.state.status = RunStatus::Blocked;
15321 runner.state.candidates[0].failed = Some("implementation failed".into());
15322 runner.state.save().unwrap();
15323 let error = runner.execute_graph().await.unwrap_err();
15324 assert!(
15325 error
15326 .to_string()
15327 .contains("selected implementation is held")
15328 );
15329 }
15330
15331 #[tokio::test]
15332 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15333 let (_tmp, mut runner) = preservation_resume_fixture().await;
15334 let repo = runner.state.repo.clone();
15335 let candidate = runner.state.candidates[0].clone();
15336 let registered = git::worktree_holding(&repo, &candidate.branch)
15339 .await
15340 .unwrap()
15341 .unwrap();
15342 assert_eq!(
15343 registered.canonicalize().unwrap(),
15344 candidate.worktree.canonicalize().unwrap()
15345 );
15346 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15347 assert_eq!(
15348 git::worktree_holding(&repo, &candidate.branch)
15349 .await
15350 .unwrap(),
15351 Some(registered)
15352 );
15353 assert!(
15354 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15355 "the stale registration initially keeps the branch checked out"
15356 );
15357 runner.fold_losers().await.unwrap();
15358 assert!(runner.state.candidates[0].folded);
15359 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15360 assert_eq!(
15361 git::worktree_holding(&repo, &candidate.branch)
15362 .await
15363 .unwrap(),
15364 None
15365 );
15366 runner.state = RunState::load(&runner.state.id).unwrap();
15367 runner.fold_losers().await.unwrap();
15368 }
15369
15370 #[tokio::test]
15371 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15372 let (_tmp, mut runner) = preservation_resume_fixture().await;
15373 let repo = runner.state.repo.clone();
15374 let candidate = runner.state.candidates[0].clone();
15375 let winner = runner.state.candidates[1].clone();
15376 let registered = git::worktree_holding(&repo, &candidate.branch)
15377 .await
15378 .unwrap()
15379 .unwrap();
15380 assert_eq!(
15381 registered.canonicalize().unwrap(),
15382 candidate.worktree.canonicalize().unwrap()
15383 );
15384 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15385 assert_eq!(
15386 git::worktree_holding(&repo, &candidate.branch)
15387 .await
15388 .unwrap(),
15389 Some(registered)
15390 );
15391 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15392
15393 for _ in 0..2 {
15394 fold_run(&mut runner.state, false, &crate::run::home())
15395 .await
15396 .unwrap();
15397 runner.state = RunState::load(&runner.state.id).unwrap();
15398 assert!(runner.state.candidates[0].folded);
15399 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15400 assert_eq!(
15401 git::worktree_holding(&repo, &candidate.branch)
15402 .await
15403 .unwrap(),
15404 None
15405 );
15406 assert!(!runner.state.candidates[1].folded);
15407 assert!(winner.worktree.exists());
15408 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15409 }
15410 }
15411
15412 #[tokio::test]
15413 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15414 ask_test_home();
15415 let tmp = tempfile::tempdir().unwrap();
15416 let repo = tmp.path().join("repo");
15417 std::fs::create_dir(&repo).unwrap();
15418 init_repo(&repo);
15419 let mut runner = runner_at(RunStatus::Implementing);
15420 runner.state.repo = repo.clone();
15421 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15422 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15423 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15424 let branch = format!("candidate-{label}");
15425 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15426 git::worktree_add_branch(&repo, &wt, &branch, "main")
15427 .await
15428 .unwrap();
15429 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15430 git::git(&wt, &["add", "README.md"]).await.unwrap();
15431 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15432 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15433 runner.state.candidates.push(Candidate {
15434 index,
15435 label,
15436 agent: "alpha".to_owned(),
15437 branch,
15438 worktree: wt,
15439 summary: "completed implementation".to_owned(),
15440 stat: String::new(),
15441 files: 0,
15442 commits: 0,
15443 empty: false,
15444 failed: None,
15445 verified_noop: None,
15446 duration_ms: 42,
15447 folded: false,
15448 });
15449 runner
15450 .state
15451 .candidate_preservation
15452 .insert(label, "awaiting persistence".to_owned());
15453 }
15454 let wt = runner.state.candidates[0].worktree.clone();
15455 let index_lock = git::git(
15456 &wt,
15457 &[
15458 "rev-parse",
15459 "--path-format=absolute",
15460 "--git-path",
15461 "index.lock",
15462 ],
15463 )
15464 .await
15465 .unwrap();
15466 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15467 runner.state.save().unwrap();
15468 assert!(runner.persist_candidates().await.is_err());
15469 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15470 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15471 assert_eq!(
15472 runner.state.candidates[1].commits, 1,
15473 "other results are still collected"
15474 );
15475 assert!(
15476 !runner.state.candidates[0].empty,
15477 "failed rescue is not no change"
15478 );
15479 assert_eq!(
15480 runner.state.candidates[0].summary,
15481 "completed implementation"
15482 );
15483 assert!(runner.after_implement().is_err());
15484 let home = tmp.path().join("fold-home");
15485 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15486 assert!(wt.join("new-file").exists());
15487 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15488 assert!(recovery.join("staged.patch").exists());
15489 assert!(recovery.join("unstaged.patch").exists());
15490 assert_eq!(
15491 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15492 "untracked edits\n"
15493 );
15494 runner.state.begin_pass();
15495 runner.state.end_pass_parked(ParkCause::Upgrade {
15496 from: "1.0".into(),
15497 to: Some("1.1".into()),
15498 });
15499 runner.state.save().unwrap();
15500 runner.state = RunState::load(&runner.state.id).unwrap();
15501 assert!(matches!(
15502 runner.state.passes.last().unwrap().end,
15503 Some(crate::run::PassEnd::Parked {
15504 cause: ParkCause::Upgrade { .. }
15505 })
15506 ));
15507 std::fs::remove_file(index_lock).unwrap();
15508 runner.implement().await.unwrap();
15510 assert!(runner.state.candidate_preservation.is_empty());
15511 assert!(
15512 runner
15513 .state
15514 .candidates
15515 .iter()
15516 .all(|c| c.commits == 1 && !c.empty)
15517 );
15518 assert!(git::is_clean(&wt).await.unwrap());
15519 assert!(
15520 crate::run::read_artifact(&runner.state, "cand-A.patch")
15521 .unwrap()
15522 .contains("later tracked edits")
15523 );
15524 }
15525
15526 #[tokio::test]
15527 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15528 ask_test_home();
15529 let tmp = tempfile::tempdir().unwrap();
15530 let repo = tmp.path().join("repo");
15531 std::fs::create_dir(&repo).unwrap();
15532 init_repo(&repo);
15533 let wt = tmp.path().join("candidate");
15534 git::worktree_add_branch(&repo, &wt, "candidate", "main")
15535 .await
15536 .unwrap();
15537 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15538 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15539 assert!(wt.join("uncommitted").exists());
15540 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15541 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15542 let mut runner = runner_at(RunStatus::Implementing);
15543 runner.state.repo = repo.clone();
15544 runner.state.base_commit = "missing-base".to_owned();
15545 runner.state.candidates.push(Candidate {
15546 index: 0,
15547 label: 'A',
15548 agent: "alpha".into(),
15549 branch: "candidate".into(),
15550 worktree: wt.clone(),
15551 summary: "completed".into(),
15552 stat: String::new(),
15553 files: 0,
15554 commits: 0,
15555 empty: false,
15556 failed: None,
15557 verified_noop: None,
15558 duration_ms: 1,
15559 folded: false,
15560 });
15561 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15562 assert!(
15563 fold_run(&mut runner.state, true, &tmp.path().join("home"))
15564 .await
15565 .is_err()
15566 );
15567 assert!(!runner.state.candidates[0].folded);
15568 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15569 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15570 runner
15571 .state
15572 .candidate_preservation
15573 .insert('A', "pending".into());
15574 assert!(runner.persist_candidates().await.is_err());
15575 assert!(!runner.state.candidates[0].empty);
15576 assert!(!runner.state.candidates[0].folded);
15577 assert!(wt.exists());
15578 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15579 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15580 std::fs::create_dir_all(&patch_path).unwrap();
15581 assert!(
15582 runner.persist_candidates().await.is_err(),
15583 "artifact failure stays pending"
15584 );
15585 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15586 assert!(!runner.state.candidates[0].empty);
15587 std::fs::remove_dir(patch_path).unwrap();
15588 runner.persist_candidates().await.unwrap();
15589 assert!(
15590 runner.state.candidates[0].empty,
15591 "only successful inspection confirms no change"
15592 );
15593 }
15594
15595 fn init_repo(dir: &Path) {
15596 let run = |args: &[&str]| {
15597 let out = std::process::Command::new("git")
15598 .args(args)
15599 .current_dir(dir)
15600 .quiet()
15601 .output()
15602 .expect("spawn git");
15603 assert!(
15604 out.status.success(),
15605 "git {args:?} failed: {}",
15606 String::from_utf8_lossy(&out.stderr)
15607 );
15608 };
15609 run(&["init", "-b", "main"]);
15610 run(&["config", "user.name", "magi test"]);
15611 run(&["config", "user.email", "magi@example.com"]);
15612 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15613 run(&["add", "-A"]);
15614 run(&["commit", "-m", "init"]);
15615 }
15616
15617 fn ask_test_home() {
15625 crate::run::pin_test_home();
15626 }
15627
15628 #[cfg(unix)]
15629 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15630 ask_test_home();
15631 let tmp = tempfile::tempdir().unwrap();
15632 let repo = tmp.path().join("repo");
15633 std::fs::create_dir(&repo).unwrap();
15634 init_repo(&repo);
15635 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15636 let origin = tmp.path().join("origin.git");
15637 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15638 .await
15639 .unwrap();
15640 git::git(
15641 &repo,
15642 &["remote", "add", "origin", origin.to_str().unwrap()],
15643 )
15644 .await
15645 .unwrap();
15646 git::git(&repo, &["checkout", "-b", "fix/retained"])
15647 .await
15648 .unwrap();
15649 std::fs::write(repo.join("retained"), "retained change").unwrap();
15650 git::git(&repo, &["add", "."]).await.unwrap();
15651 git::git(&repo, &["commit", "-m", "retained implementation"])
15652 .await
15653 .unwrap();
15654 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15655 git::git(&repo, &["checkout", "main"]).await.unwrap();
15656 std::fs::write(repo.join("base-change"), "new base").unwrap();
15657 git::git(&repo, &["add", "."]).await.unwrap();
15658 git::git(&repo, &["commit", "-m", "base moved"])
15659 .await
15660 .unwrap();
15661 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15662 git::git(&repo, &["checkout", "fix/retained"])
15663 .await
15664 .unwrap();
15665 let mut runner = runner_at(RunStatus::Voting);
15666 runner.state.repo = repo.clone();
15667 runner.state.base_commit = base;
15668 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15669 runner.state.config.disk.min_free_bytes = 0;
15670 runner.state.config.graph.review_rounds = 1;
15671 runner.state.config.verify.e2e = vec!["true".into()];
15672 runner.state.config.verify.gate = vec!["true".into()];
15673 runner.state.candidates = vec![Candidate {
15674 index: 0,
15675 label: 'A',
15676 agent: "fixture".into(),
15677 branch: "fix/retained".into(),
15678 worktree: repo.clone(),
15679 summary: String::new(),
15680 stat: String::new(),
15681 files: 1,
15682 commits: 1,
15683 empty: false,
15684 failed: None,
15685 verified_noop: None,
15686 duration_ms: 0,
15687 folded: false,
15688 }];
15689 runner.state.tally = Some(Tally {
15690 first_choice: BTreeMap::new(),
15691 borda: BTreeMap::new(),
15692 winner: 'A',
15693 rankings: 0,
15694 unanimous_initial: true,
15695 deliberated: false,
15696 changed_votes: 0,
15697 unanimous_final: true,
15698 tie_break: None,
15699 judges: 0,
15700 present: 0,
15701 quorum: 0,
15702 met_quorum: true,
15703 uncontested: Some("review-only".into()),
15704 });
15705 let mut task = crate::queue::Task::new(
15706 "fixture".into(),
15707 "fixture".into(),
15708 repo,
15709 crate::queue::Source::Human,
15710 );
15711 task.link_run(&runner.state.id);
15712 crate::queue::Queue::open().put(&mut task).unwrap();
15713 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15714 runner.state.base_sync = Some(BaseSync {
15715 tip: String::new(),
15716 behind: 1,
15717 attempts: BASE_SYNC_ROUNDS,
15718 conflict: None,
15719 already_in: None,
15720 });
15721 runner.sync_to_base().await.unwrap();
15722 assert_eq!(runner.state.status, RunStatus::Blocked);
15723 (tmp, runner, head, task.id)
15724 }
15725
15726 #[cfg(unix)]
15727 #[tokio::test]
15728 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15729 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15730 if std::env::var_os(CHILD).is_none() {
15731 let home = tempfile::tempdir().unwrap();
15732 let output = std::process::Command::new(std::env::current_exe().unwrap())
15733 .args([
15734 "--exact",
15735 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15736 "--nocapture",
15737 ])
15738 .env(CHILD, "1")
15739 .env("MAGI_HOME", home.path())
15740 .env("MAGI_WORKER_HOME", home.path())
15741 .env("MAGI_WORKER_ID", "abcdef")
15742 .output()
15743 .unwrap();
15744 assert!(
15745 output.status.success()
15746 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15747 "{}\n{}",
15748 String::from_utf8_lossy(&output.stdout),
15749 String::from_utf8_lossy(&output.stderr)
15750 );
15751 return;
15752 }
15753 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15754 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15755 let home = crate::run::home();
15756 let queue = crate::queue::Queue::open();
15757 let mut owner = queue.get(&task).unwrap();
15758 owner.release_operator("test-release", None);
15759 owner.start(runner.state.id.clone());
15760 queue.put(&mut owner).unwrap();
15761 runner.state.driver_exited = true;
15762 runner.state.task_instruction = Some(owner.instruction.clone());
15763 runner.state.save().unwrap();
15764 let _claim = queue.claim(&task).unwrap();
15765 let reserve = |id: &str| {
15766 let directory = home.join("workers/jobs").join(id);
15767 std::fs::create_dir_all(&directory).unwrap();
15768 let job = crate::worker::Job {
15769 id: id.into(),
15770 sequence: 1,
15771 kind: "task".into(),
15772 key: format!("task:{task}"),
15773 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15774 generation: "fixture-generation".into(),
15775 version: None,
15776 status: crate::worker::Status::Running,
15777 result: None,
15778 pid: Some(std::process::id()),
15779 created_at: 0,
15780 finished_at: None,
15781 };
15782 std::fs::write(
15783 directory.join("job.json"),
15784 serde_json::to_vec(&job).unwrap(),
15785 )
15786 .unwrap();
15787 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15788 };
15789 let _own = reserve("abcdef");
15790 let other = reserve("fedcba");
15791 assert!(
15792 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15793 .await
15794 .is_err()
15795 );
15796 assert!(
15797 RunState::load(&runner.state.id)
15798 .unwrap()
15799 .base_sync_recovery
15800 .authorizations
15801 .is_empty()
15802 );
15803 drop(other);
15804 assert!(crate::daemon::is_working_on(
15805 &home,
15806 &runner.state.id,
15807 Timestamp::now()
15808 ));
15809 assert!(
15810 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15811 );
15812 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15813 .await
15814 .unwrap();
15815 assert!(
15816 RunState::load(&runner.state.id)
15817 .unwrap()
15818 .base_recovery_pending()
15819 );
15820 }
15821
15822 #[cfg(unix)]
15823 #[tokio::test]
15824 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15825 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15826 let home = crate::run::home();
15827 runner.state.driver_pid = Some(std::process::id());
15828 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15829 runner.state.driver_exited = false;
15830 runner.state.begin_pass();
15831 runner.state.save().unwrap();
15832 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15833 driver.record(&runner.state).unwrap();
15834 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15835 drop(driver);
15837 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15838 assert_eq!(
15839 runner.state.liveness_under(&home, false),
15840 crate::run::Liveness::Dead
15841 );
15842 let queue = crate::queue::Queue::open();
15843 let _claim = queue.claim(&task).unwrap();
15844 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15845 .await
15846 .unwrap();
15847 let disk = RunState::load(&runner.state.id).unwrap();
15848 assert!(!disk.driver_exited);
15849 assert!(disk.base_recovery_pending());
15850 }
15851
15852 #[cfg(unix)]
15853 #[tokio::test]
15854 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15855 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15856 if std::env::var_os(CHILD).is_none() {
15857 use std::os::unix::fs::PermissionsExt;
15858 let tmp = tempfile::tempdir().unwrap();
15859 let gh = tmp.path().join("gh");
15860 std::fs::write(&gh, r#"#!/bin/sh
15861case "$1 $2" in
15862 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15863 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15864 *) exit 2 ;;
15865esac
15866"#).unwrap();
15867 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15868 let mut paths = vec![tmp.path().to_path_buf()];
15869 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15870 let output = std::process::Command::new(std::env::current_exe().unwrap())
15871 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15872 .env(CHILD, "1")
15873 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15874 .env("PATH", std::env::join_paths(paths).unwrap())
15875 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15876 .output().unwrap();
15877 assert!(
15878 output.status.success()
15879 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15880 "{}\n{}",
15881 String::from_utf8_lossy(&output.stdout),
15882 String::from_utf8_lossy(&output.stderr)
15883 );
15884 return;
15885 }
15886 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15887 let repo = runner.state.repo.clone();
15888 git::git(&repo, &["checkout", "main"]).await.unwrap();
15889 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15890 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15891 git::git(&repo, &["checkout", "fix/retained"])
15892 .await
15893 .unwrap();
15894 runner.state.pr = Some(crate::run::PrRecord {
15895 url: "https://example.test/project/pull/1".into(),
15896 number: 1,
15897 state: "open".into(),
15898 checks: "unknown".into(),
15899 round: 0,
15900 rounds: 1,
15901 red_at_merge: Vec::new(),
15902 });
15903 runner.sync_to_base().await.unwrap();
15904 assert_eq!(runner.state.status, RunStatus::Blocked);
15905 assert_eq!(
15906 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15907 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15908 );
15909 assert!(runner.state.base_sync_closure_pending());
15910 assert!(crate::handover::can_resume(&runner.state));
15911 let queue = crate::queue::Queue::open();
15912 assert!(!crate::base_sync_recovery::held_task(
15913 &queue,
15914 &queue.get(&task).unwrap()
15915 ));
15916 assert!(runner.state.authorize_base_sync(&task).is_err());
15917 runner.state.base_sync_recovery.stop = None;
15919 runner.state.status = RunStatus::Voting;
15920 runner.state.gate_ran = true;
15921 runner.state.config.graph.review_rounds = 0;
15922 runner.state.save().unwrap();
15923 assert!(crate::handover::can_resume(&runner.state));
15924 runner.state = RunState::load(&runner.state.id).unwrap();
15925 runner.execute_graph().await.unwrap();
15926 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15927 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15928 assert_eq!(
15929 runner.state.base_sync.as_ref().unwrap().attempts,
15930 BASE_SYNC_ROUNDS
15931 );
15932 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15933 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15934 assert!(runner.state.reviews.is_empty());
15935 assert!(runner.state.gate.is_empty());
15936 }
15937
15938 #[cfg(unix)]
15939 #[tokio::test]
15940 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15941 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15942 runner.state.driver_exited = true;
15943 runner.state.save().unwrap();
15944 let home = crate::run::home();
15945 let queue = crate::queue::Queue::open();
15946 let _claim = queue.claim(&task).unwrap();
15947 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15948 assert!(
15949 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15950 .await
15951 .is_err()
15952 );
15953 drop(lease);
15954 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15955 assert!(
15956 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15957 .await
15958 .is_err()
15959 );
15960 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15961 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15962 .await
15963 .unwrap();
15964 let disk = RunState::load(&runner.state.id).unwrap();
15965 assert!(disk.base_recovery_pending());
15966 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15967 assert!(
15968 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15969 .await
15970 .is_err()
15971 );
15972 assert_eq!(
15973 RunState::load(&runner.state.id)
15974 .unwrap()
15975 .base_sync_recovery
15976 .authorizations
15977 .len(),
15978 1
15979 );
15980 let mut owner = queue.get(&task).unwrap();
15981 owner.runs.push("20260101-000000-ffff".into());
15982 queue.put(&mut owner).unwrap();
15983 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15984 .await
15985 .unwrap_err();
15986 assert!(error.to_string().contains("run task ownership changed"));
15987 }
15988
15989 #[cfg(unix)]
15990 #[tokio::test]
15991 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15992 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15993 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15994 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15995 legacy["status"] = serde_json::json!("voting");
15996 runner.state = serde_json::from_value(legacy).unwrap();
15997 runner.state.save().unwrap();
15998 for _ in 0..3 {
15999 runner.execute_graph().await.unwrap();
16000 assert_eq!(runner.state.status, RunStatus::Blocked);
16001 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16002 assert!(!crate::handover::can_resume(&runner.state));
16003 assert_eq!(
16004 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16005 head
16006 );
16007 assert!(runner.state.reviews.is_empty());
16008 assert!(!runner.state.gate_ran);
16009 }
16010 let notice = crate::notices::run_ended(&runner.state).unwrap();
16011 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16012 runner.state.begin_pass();
16013 runner.state.status = RunStatus::Voting;
16014 runner.deliberate().await.unwrap();
16015 runner.vote().await.unwrap();
16016 runner.finish_after_tally().await.unwrap();
16017 assert_eq!(runner.state.status, RunStatus::Blocked);
16018 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16019 assert_eq!(
16020 crate::notices::run_ended(&runner.state).unwrap().since,
16021 notice.since
16022 );
16023 }
16024
16025 #[cfg(unix)]
16026 #[tokio::test]
16027 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16028 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16029 for detail in [
16030 "fix/retained conflicts with origin/main and did not rebase: conflict",
16031 "rebased fix/retained locally but origin refused the push",
16032 "unknown failure",
16033 ] {
16034 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16035 runner.state.record_base_sync_stop(&head);
16036 assert!(runner.state.authorize_base_sync(&task).is_err());
16037 runner.state.status = RunStatus::Voting;
16038 runner.execute_graph().await.unwrap();
16039 assert_eq!(runner.state.status, RunStatus::Blocked);
16040 assert!(!crate::handover::can_resume(&runner.state));
16041 assert_eq!(
16042 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16043 head
16044 );
16045 }
16046 }
16047
16048 #[cfg(unix)]
16049 #[tokio::test]
16050 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16051 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16052 runner.state.authorize_base_sync(&task).unwrap();
16053 assert!(runner.state.consume_base_recovery(&head));
16054 runner.state.save().unwrap();
16055 runner.state = RunState::load(&runner.state.id).unwrap();
16056 runner.execute_graph().await.unwrap();
16057 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16058 assert_eq!(runner.state.status, RunStatus::Blocked);
16059 assert!(!runner.state.consume_base_recovery(&head));
16060 assert!(!crate::handover::can_resume(&runner.state));
16061 runner.state.authorize_base_sync(&task).unwrap();
16062 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
16063 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16064 assert_eq!(
16065 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16066 head
16067 );
16068 }
16069
16070 #[cfg(unix)]
16071 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
16072 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
16073 runner.state.prep_complete = Some(true);
16075 runner.state.base_sync = None;
16076 runner.state.base_sync_recovery = Default::default();
16077 runner.state.config.graph.gate_fix_rounds = 1;
16078 let mut review = review_round(true, 0, 1, 1, false, true);
16079 review.head = head.clone();
16080 review.verified_head = Some(head.clone());
16081 runner.state.reviews = vec![review];
16082 runner.state.gate = vec![CommandOutcome {
16083 command: "old gate".into(),
16084 code: Some(1),
16085 output_tail: "old failure".into(),
16086 duration_ms: 0,
16087 resource_blocked: false,
16088 }];
16089 runner.state.gate_head = Some(head.clone());
16090 runner.state.gate_ran = true;
16091 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
16092 agent: "fixture".into(),
16093 failed: runner.state.gate.clone(),
16094 notes: "spent".into(),
16095 committed: false,
16096 error: None,
16097 }];
16098 runner.state.land_approval = Some(crate::run::LandApproval {
16099 question: "old approval".into(),
16100 head: head.clone(),
16101 });
16102 runner.state.status = RunStatus::Blocked;
16103 (tmp, runner, head)
16104 }
16105
16106 #[cfg(unix)]
16107 #[tokio::test]
16108 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
16109 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
16110 let (tmp, mut runner, old) = resumed_gate_fixture().await;
16111 let phases = tmp.path().join("phases");
16112 runner.state.config.verify.e2e = vec![format!(
16113 "echo e2e >> '{}'; {}",
16114 phases.display(),
16115 if e2e_ok {
16116 "true"
16117 } else {
16118 "echo new-e2e-failure; false"
16119 }
16120 )];
16121 runner.state.config.verify.gate = vec![format!(
16122 "echo gate >> '{}'; {}",
16123 phases.display(),
16124 if gate_ok {
16125 "true"
16126 } else {
16127 "echo new-gate-failure; false"
16128 }
16129 )];
16130 runner.state.save().unwrap();
16131 runner.state = RunState::load(&runner.state.id).unwrap();
16132 runner.sync_to_base().await.unwrap();
16133 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16134 assert_ne!(head, old);
16135 runner.state = RunState::load(&runner.state.id).unwrap();
16137 runner.execute_graph().await.unwrap();
16138 assert_eq!(runner.state.gate_fixes.len(), 1);
16139 assert_eq!(runner.state.reviews.len(), 1);
16140 assert_eq!(runner.state.reviews[0].head, old);
16141 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
16142 assert_eq!(runner.state.verification_history.len(), 1);
16143 let history = &runner.state.verification_history[0];
16144 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
16145 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16146 assert!(history.archived_at.is_some());
16147 assert_eq!(history.fixes_spent, Some(1));
16148 assert_eq!(history.gate[0].output_tail, "old failure");
16149 let archive_events: Vec<_> = runner
16150 .state
16151 .events
16152 .iter()
16153 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
16154 .collect();
16155 assert_eq!(archive_events.len(), 1);
16156 assert!(archive_events[0].message.contains(&short(&old)));
16157 assert!(archive_events[0].message.contains(&short(&head)));
16158 assert!(
16159 archive_events[0]
16160 .message
16161 .contains("1 gate-fix round(s) already spent stay spent")
16162 );
16163 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
16164 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
16165 if e2e_ok {
16166 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16167 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
16168 } else {
16169 assert!(!runner.state.gate_ran);
16170 }
16171 assert_eq!(
16172 runner.state.status,
16173 if e2e_ok && gate_ok {
16174 RunStatus::Ready
16175 } else {
16176 RunStatus::Blocked
16177 }
16178 );
16179 if !e2e_ok || !gate_ok {
16180 let cause = runner.state.verification_blocking_cause().unwrap();
16181 assert!(cause.contains(&head));
16182 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
16183 runner.state.event("land", "rebased successfully");
16185 let notice = crate::notices::run_ended(&runner.state).unwrap();
16186 assert!(!format!("{notice:?}").contains("rebased successfully"));
16187 assert!(runner.state.merge.is_none());
16188 }
16189 let before = std::fs::read_to_string(&phases).unwrap();
16190 runner.state.save().unwrap();
16191 runner.state = RunState::load(&runner.state.id).unwrap();
16192 runner.review_loop().await.unwrap();
16193 runner.gate().await.unwrap();
16194 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
16195 assert_eq!(runner.state.verification_history.len(), 1);
16196 let history = &runner.state.verification_history[0];
16197 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
16198 assert!(history.archived_at.is_some());
16199 assert_eq!(history.fixes_spent, Some(1));
16200 }
16201 }
16202
16203 #[cfg(unix)]
16204 #[tokio::test]
16205 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
16206 let (tmp, mut runner, head) = resumed_gate_fixture().await;
16207 let marker = tmp.path().join("gate-ran");
16208 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
16209 runner.review_loop().await.unwrap();
16210 runner.gate().await.unwrap();
16211 assert_eq!(runner.state.status, RunStatus::Blocked);
16212 assert!(!marker.exists());
16213 assert!(runner.state.verification_history.is_empty());
16214 runner.state.gate_head = None;
16215 runner.gate().await.unwrap();
16216 assert!(marker.exists());
16217 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16218 assert_eq!(runner.state.verification_history.len(), 1);
16219 assert_eq!(runner.state.gate_fixes.len(), 1);
16220 }
16221
16222 #[cfg(unix)]
16223 #[tokio::test]
16224 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
16225 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16226 runner.sync_to_base().await.unwrap();
16227 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16228 runner.state.reconcile_verification_head(&head);
16229 for outcomes in [
16231 Vec::new(),
16232 vec![CommandOutcome {
16233 command: "true".into(),
16234 code: None,
16235 output_tail: "cache unavailable".into(),
16236 duration_ms: 0,
16237 resource_blocked: true,
16238 }],
16239 ] {
16240 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16241 head: head.clone(),
16242 passed: false,
16243 outcomes,
16244 });
16245 runner.state.gate_ran = false;
16246 runner.state.gate.clear();
16247 runner.state.save().unwrap();
16248 runner.state = RunState::load(&runner.state.id).unwrap();
16249 runner.merge().await.unwrap();
16250 assert!(runner.state.merge.is_none());
16251 runner.review_loop().await.unwrap();
16252 runner.gate().await.unwrap();
16253 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
16254 assert!(runner.state.gate_allows_progress());
16255 assert_eq!(runner.state.gate_fixes.len(), 1);
16256 assert!(runner.state.merge.is_none());
16257 }
16258 git::git(
16260 &runner.state.repo,
16261 &["commit", "--allow-empty", "-m", "external update"],
16262 )
16263 .await
16264 .unwrap();
16265 runner.merge().await.unwrap();
16266 assert!(runner.state.merge.is_none());
16267 assert_eq!(runner.state.status, RunStatus::Blocked);
16268 }
16269
16270 #[cfg(unix)]
16271 #[tokio::test]
16272 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16273 for waiting_on_gate in [false, true] {
16274 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16275 runner.sync_to_base().await.unwrap();
16276 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16277 runner.state.reconcile_verification_head(&head);
16278 if waiting_on_gate {
16279 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16280 head: head.clone(),
16281 passed: true,
16282 outcomes: Vec::new(),
16283 });
16284 }
16285 let cache = tmp.path().join("cache");
16286 runner.state.config.verify.e2e =
16287 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16288 runner.state.config.verify.gate = vec!["true".into()];
16289 runner.state.config.graph.timeout_resource_wait = 1;
16290 let owner =
16291 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16292 let guard =
16293 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16294 crate::cache::AcquireOutcome::Acquired(g) => g,
16295 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16296 };
16297 runner.review_loop().await.unwrap();
16298 runner.gate().await.unwrap();
16299 assert!(!runner.state.gate_ran);
16300 assert_eq!(runner.state.gate_fixes.len(), 1);
16301 assert!(runner.state.resource_block.is_some());
16302 runner.merge().await.unwrap();
16303 assert!(runner.state.merge.is_none());
16304 drop(guard);
16305 runner.state = RunState::load(&runner.state.id).unwrap();
16306 runner.state.resource_block = None;
16308 runner.review_loop().await.unwrap();
16309 runner.gate().await.unwrap();
16310 assert!(runner.state.gate_allows_progress());
16311 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16312 assert_eq!(runner.state.gate_fixes.len(), 1);
16313 }
16314 }
16315
16316 #[cfg(unix)]
16317 #[tokio::test]
16318 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16319 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16320 runner.sync_to_base().await.unwrap();
16321 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16322 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16323 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16325 runner.review_loop().await.unwrap();
16326 runner.gate().await.unwrap();
16327 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16328 assert_ne!(before, after);
16329 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16330 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16331 assert!(!runner.state.gate_allows_progress());
16332 runner.merge().await.unwrap();
16333 assert!(runner.state.merge.is_none());
16334 assert!(
16335 runner
16336 .state
16337 .verification_history
16338 .iter()
16339 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16340 );
16341
16342 runner.state.reviews[0].clean = false;
16344 runner.state.reviews[0].answered = 0;
16345 runner.state.config.verify.e2e = vec!["true".into()];
16346 runner.review_loop().await.unwrap();
16347 runner.gate().await.unwrap();
16348 runner.merge().await.unwrap();
16349 assert_eq!(runner.state.status, RunStatus::Blocked);
16350 assert!(runner.state.merge.is_none());
16351 assert!(
16352 runner
16353 .state
16354 .verification_blocking_cause()
16355 .unwrap()
16356 .contains("review blocked")
16357 );
16358 }
16359
16360 #[cfg(unix)]
16361 #[tokio::test]
16362 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16363 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16364 runner.state.gate_head = None;
16365 runner.state.config.verify.gate =
16366 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16367 runner.review_loop().await.unwrap();
16368 runner.gate().await.unwrap();
16369 assert_ne!(
16370 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16371 old
16372 );
16373 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16374 assert!(!runner.state.gate_allows_progress());
16375 assert_eq!(runner.state.status, RunStatus::Blocked);
16376 runner.merge().await.unwrap();
16377 assert!(runner.state.merge.is_none());
16378 }
16379
16380 #[cfg(unix)]
16381 #[tokio::test]
16382 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16383 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16384 let mut old_review = review_round(true, 0, 1, 1, false, true);
16385 old_review.head = head.clone();
16386 old_review.verified_head = Some(head.clone());
16387 runner.state.reviews = vec![old_review];
16388 runner.state.gate = vec![CommandOutcome {
16389 command: "old gate".into(),
16390 code: None,
16391 output_tail: "timed out after 2400s".into(),
16392 duration_ms: 2_400_000,
16393 resource_blocked: false,
16394 }];
16395 runner.state.gate_ran = true;
16396 runner.state.gate_head = Some(head.clone());
16397 runner.state.land_approval = Some(crate::run::LandApproval {
16398 question: "old-head-approval".into(),
16399 head: head.clone(),
16400 });
16401 runner.state.authorize_base_sync(&task).unwrap();
16402 runner.sync_to_base().await.unwrap();
16403 assert!(!runner.state.base_sync_blocked());
16404 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16405 assert_ne!(head, fresh);
16406 assert!(runner.state.reviews.is_empty());
16407 assert!(!runner.state.gate_ran);
16408 let saved = &runner.state.base_sync_recovery.authorizations[0];
16409 assert_eq!(saved.reviews[0].head, head);
16410 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16411 assert!(saved.gate_ran);
16412 assert!(!saved.gate[0].ok());
16413 runner.review_loop().await.unwrap();
16414 runner.gate().await.unwrap();
16415 assert_eq!(runner.state.reviews[0].head, fresh);
16416 assert_eq!(
16417 runner.state.reviews[0].verified_head.as_deref(),
16418 Some(fresh.as_str())
16419 );
16420 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16421 assert!(runner.state.gate_ran);
16422 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16423 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16424 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16425 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16426 assert_eq!(
16427 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16428 "retained change"
16429 );
16430 }
16431
16432 #[cfg(unix)]
16433 #[tokio::test]
16434 async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16435 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16436 let queue = crate::queue::Queue::open();
16437 let mut owner = queue.get(&task).unwrap();
16438 owner.release_operator("test-resume", None);
16439 queue.put(&mut owner).unwrap();
16440 let generation = owner.resumption.as_ref().unwrap().generation;
16441 runner.state.task_instruction = Some(owner.instruction.clone());
16442 runner.state.driver_exited = true;
16443 runner.state.save().unwrap();
16444 let home = crate::run::home();
16445 let _claim = queue.claim(&task).unwrap();
16446 owner.resumption.as_mut().unwrap().at =
16447 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16448 queue.put(&mut owner).unwrap();
16449 assert!(
16450 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16451 .await
16452 .is_err(),
16453 "an approval from before the stop grants nothing"
16454 );
16455 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16456 queue.put(&mut owner).unwrap();
16457 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16458 .await
16459 .unwrap();
16460 assert!(
16461 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16462 .await
16463 .is_err()
16464 );
16465 runner.state = RunState::load(&runner.state.id).unwrap();
16466 assert_eq!(
16467 runner.state.base_sync_recovery.authorizations[0].generation,
16468 Some(generation)
16469 );
16470 runner.sync_to_base().await.unwrap();
16471 let repo = runner.state.repo.clone();
16472 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16473 git::git(&repo, &["checkout", "main"]).await.unwrap();
16474 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16475 git::git(&repo, &["add", "."]).await.unwrap();
16476 git::git(&repo, &["commit", "-m", "move base again"])
16477 .await
16478 .unwrap();
16479 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16480 git::git(&repo, &["checkout", "fix/retained"])
16481 .await
16482 .unwrap();
16483 runner.finish_after_tally().await.unwrap();
16484 assert_eq!(runner.state.status, RunStatus::Blocked);
16485 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16486 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16487 assert!(!runner.state.base_recovery_pending());
16488 assert!(!crate::handover::can_resume(&runner.state));
16489 assert!(!runner.state.gate_ran);
16490 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16492 queue.put(&mut owner).unwrap();
16493 assert!(
16494 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16495 .await
16496 .is_err()
16497 );
16498 runner.state = RunState::load(&runner.state.id).unwrap();
16499 runner.sync_to_base().await.unwrap();
16500 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16501 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16502 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16503 }
16504
16505 #[cfg(unix)]
16506 #[tokio::test]
16507 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16508 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16509 runner.state.base_sync = None;
16510 runner.state.status = RunStatus::Blocked;
16511 let mut second = runner.state.candidates[0].clone();
16512 second.label = 'B';
16513 runner.state.candidates.push(second);
16514 assert!(runner.state.votes.is_empty());
16515 assert!(runner.state.deliberation.is_empty());
16516 runner.deliberate().await.unwrap();
16517 runner.vote().await.unwrap();
16518 assert_eq!(runner.state.status, RunStatus::Blocked);
16519 assert!(runner.state.votes.is_empty());
16520 assert!(runner.state.deliberation.is_empty());
16521 }
16522
16523 fn runner_at(status: RunStatus) -> Runner {
16526 let mut state = RunState::new(
16527 PathBuf::from("/nonexistent/repo"),
16528 "main".to_owned(),
16529 "deadbeef".to_owned(),
16530 "task".to_owned(),
16531 Config::default(),
16532 );
16533 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16536 state.id = format!(
16537 "20000101-000000-{:04x}",
16538 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16539 );
16540 state.status = status;
16541 Runner {
16542 state,
16543 roles: ResolvedRoles {
16544 implementers: Vec::new(),
16545 judges: Vec::new(),
16546 reviewers: Vec::new(),
16547 fixer: None,
16548 conductor: conductor(),
16549 implementer_roster: Vec::new(),
16550 judge_roster: Vec::new(),
16551 reviewer_roster: Vec::new(),
16552 },
16553 sem: Arc::new(Semaphore::new(1)),
16554 pause: Pause::new(),
16555 interrupt: Pause::new(),
16556 }
16557 }
16558
16559 #[tokio::test]
16560 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16561 crate::run::pin_test_home();
16562 let mut runner = runner_at(RunStatus::Gating);
16563 runner.state.save().unwrap();
16564 let id = runner.state.id.clone();
16565 let passes = runner.state.passes.len();
16566 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16567 let mut claiming = Box::pin(runner.claim_driver());
16568 std::future::poll_fn(|cx| {
16569 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16570 std::task::Poll::Ready(())
16571 })
16572 .await;
16573 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16574 drop(owner);
16575 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16576 .await
16577 .unwrap()
16578 .unwrap();
16579 assert_eq!(execution.runner.state.passes.len(), passes + 1);
16580 assert!(execution.runner.state.resource_block.is_none());
16581 assert_eq!(
16582 execution.runner.state.liveness(false),
16583 crate::run::Liveness::Live
16584 );
16585 drop(execution);
16586 assert!(RunState::load(&id).unwrap().driver_exited);
16587 }
16588
16589 #[tokio::test]
16590 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16591 crate::run::pin_test_home();
16592 let mut runner = runner_at(RunStatus::Gating);
16593 runner.state.save().unwrap();
16594 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16597 assert!(runner.claim_driver().await.is_err());
16598 assert!(runner.state.driver_exited);
16599 assert_eq!(runner.state.driver_pid, None);
16600 let disk = RunState::load(&runner.state.id).unwrap();
16601 assert!(disk.driver_exited);
16602 assert_eq!(disk.driver_pid, None);
16603 assert_eq!(disk.status, RunStatus::Gating);
16604 }
16605
16606 #[tokio::test]
16607 async fn an_early_graph_error_releases_its_driver() {
16608 crate::run::pin_test_home();
16609 let mut runner = runner_at(RunStatus::Blocked);
16610 runner.state.config.graph.selected_implementer = Some("selected".into());
16611 runner.state.save().unwrap();
16612 assert!(runner.execute().await.is_err());
16613 let disk = RunState::load(&runner.state.id).unwrap();
16614 assert!(disk.driver_exited);
16615 assert_eq!(disk.driver_pid, None);
16616 assert_eq!(disk.status, RunStatus::Blocked);
16617 }
16618
16619 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16620 crate::run::MergeRepair {
16621 question: "repair-question".into(),
16622 head: "old-head".into(),
16623 pr: "https://github.com/o/r/pull/42".into(),
16624 reviews: vec![review_round(false, 1, 1, 1, false, true)],
16625 gate: vec![outcome(Some(0), false)],
16626 gate_head: Some("old-head".into()),
16627 gate_fixes: Vec::new(),
16628 pre_gate: Vec::new(),
16629 phase,
16630 failure: None,
16631 }
16632 }
16633
16634 #[tokio::test]
16635 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16636 crate::run::pin_test_home();
16637 let mut runner = runner_at(RunStatus::Landing);
16638 runner.state.merge_repairs.push(merge_repair_request(
16639 crate::run::MergeRepairPhase::Requested,
16640 ));
16641 runner.state.save().unwrap();
16642 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16643 assert!(
16644 runner
16645 .execute()
16646 .await
16647 .unwrap_err()
16648 .to_string()
16649 .contains("already running")
16650 );
16651 assert_eq!(
16652 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16653 crate::run::MergeRepairPhase::Requested
16654 );
16655 drop(claim);
16656 let mut disk = runner.state.clone();
16657 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16658 disk.save().unwrap();
16659 assert!(
16660 runner
16661 .execute()
16662 .await
16663 .unwrap_err()
16664 .to_string()
16665 .contains("decision changed")
16666 );
16667 assert_eq!(
16668 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16669 crate::run::MergeRepairPhase::Complete
16670 );
16671 }
16672
16673 #[test]
16674 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16675 crate::run::pin_test_home();
16676 let mut runner = runner_at(RunStatus::Landing);
16677 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16678 runner.state.reviews = request.reviews.clone();
16679 runner.state.gate = request.gate.clone();
16680 runner.state.gate_ran = true;
16681 runner.state.land_approval = Some(crate::run::LandApproval {
16682 question: "old".into(),
16683 head: "old-head".into(),
16684 });
16685 runner.state.merge_repairs.push(request);
16686 runner.begin_merge_repair_review(0).unwrap();
16687 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16688 assert!(!runner.state.gate_status().ok());
16689 assert!(runner.state.land_approval.is_none());
16690 let disk = RunState::load(&runner.state.id).unwrap();
16691 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16692 assert!(disk.merge_repairs[0].gate[0].ok());
16693 assert_eq!(
16694 disk.merge_repairs[0].phase,
16695 crate::run::MergeRepairPhase::Reviewing
16696 );
16697 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16698 assert!(disk.reviews.is_empty());
16699 }
16700
16701 #[tokio::test]
16702 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16703 crate::run::pin_test_home();
16704 for phase in [
16705 crate::run::MergeRepairPhase::Fixing,
16706 crate::run::MergeRepairPhase::Requested,
16707 ] {
16708 let mut runner = runner_at(RunStatus::Landing);
16709 runner.state.config.graph.land_rounds = 0;
16710 runner.state.merge_repairs.push(merge_repair_request(phase));
16711 runner.state.save().unwrap();
16712 assert!(!runner.repair_merge_review().await.unwrap());
16713 assert_eq!(runner.state.status, RunStatus::Blocked);
16714 assert_eq!(
16715 runner.state.merge_repairs[0].phase,
16716 crate::run::MergeRepairPhase::Failed
16717 );
16718 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16719 assert!(
16720 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16721 "interrupted"
16722 } else {
16723 "budget exhausted"
16724 })
16725 );
16726 assert!(runner.state.merge.is_none());
16727 }
16728 }
16729
16730 #[test]
16734 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16735 crate::run::pin_test_home();
16736 let mut runner = runner_at(RunStatus::Implementing);
16737 let interrupt = Pause::new();
16738 runner.watch_interrupt(interrupt.clone());
16739
16740 interrupt.park_because("task a1b2 asked to run first");
16741
16742 assert!(runner.park_here().expect("park_here"));
16743 assert!(runner.state.parked);
16744 let last = runner.state.events.last().expect("a park event");
16745 assert_eq!(last.node, "park");
16746 assert!(
16747 last.message.contains("task a1b2 asked to run first"),
16748 "expected the interrupt reason in {:?}",
16749 last.message
16750 );
16751 }
16752
16753 #[test]
16756 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16757 crate::run::pin_test_home();
16758 let mut runner = runner_at(RunStatus::Implementing);
16759 runner.state.begin_pass();
16760 let shutdown = Pause::new();
16761 runner.on_pause(shutdown.clone());
16762 let upgrade = ParkCause::Upgrade {
16763 from: "0.121.0".to_owned(),
16764 to: Some("0.121.1".to_owned()),
16765 };
16766 shutdown.park_for(upgrade.clone());
16767 shutdown.park_for(ParkCause::Operator);
16768
16769 assert!(runner.park_here().expect("park_here"));
16770 assert_eq!(
16771 runner.state.passes[0].end,
16772 Some(crate::run::PassEnd::Parked { cause: upgrade })
16773 );
16774 let last = runner.state.events.last().expect("a park event");
16775 assert!(last.message.contains("0.121.0"), "{}", last.message);
16776 }
16777
16778 #[tokio::test]
16779 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16780 crate::run::pin_test_home();
16781 let mut runner = runner_at(RunStatus::Reviewing);
16782 runner.state.save().unwrap();
16783 let mut execution = runner.claim_driver().await.expect("first execution");
16784 assert_eq!(execution.runner.state.passes.len(), 1);
16785 execution
16786 .driver
16787 .finish(&mut execution.runner.state)
16788 .await
16789 .unwrap();
16790 }
16791
16792 #[tokio::test]
16793 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
16794 crate::run::pin_test_home();
16795 let mut runner = runner_at(RunStatus::Reviewing);
16796 runner.state.config.agents = vec![conductor()];
16797 runner.state.config.graph.timeout_resource_wait = 0;
16798 runner.state.save().unwrap();
16799 let id = runner.state.id.clone();
16800 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16801 assert!(runner.claim_driver().await.is_err());
16802 drop(owner);
16803 drop(runner);
16804 let mut resumed = Runner::resume(&id).unwrap();
16805 assert_eq!(resumed.state.driver_claimed, Some(false));
16806 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
16807 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
16808 let mut execution = resumed
16809 .claim_driver()
16810 .await
16811 .expect("retry initial claim after reload");
16812 assert_eq!(execution.runner.state.driver_claimed, Some(true));
16813 assert_eq!(execution.runner.state.passes.len(), 1);
16814 execution
16815 .driver
16816 .finish(&mut execution.runner.state)
16817 .await
16818 .unwrap();
16819 }
16820
16821 #[tokio::test]
16822 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
16823 crate::run::pin_test_home();
16824 let mut runner = runner_at(RunStatus::Reviewing);
16825 runner.state.config.agents = vec![conductor()];
16826 runner.state.save().unwrap();
16827 let id = runner.state.id.clone();
16828 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
16829 let mut resumed = Runner::resume(&id).unwrap();
16830 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
16831 assert!(resumed.claim_driver().await.is_err());
16832 assert!(RunState::load(&id).unwrap().passes.is_empty());
16833 drop(lease);
16834 let mut execution = resumed.claim_driver().await.unwrap();
16835 execution
16836 .driver
16837 .finish(&mut execution.runner.state)
16838 .await
16839 .unwrap();
16840 }
16841
16842 #[tokio::test]
16843 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
16844 crate::run::pin_test_home();
16845 let mut runner = runner_at(RunStatus::Reviewing);
16846 let state = &mut runner.state;
16847 state.config.agents = vec![conductor()];
16848 state.driver_claimed = None;
16849 state.driver_pid = Some(std::process::id());
16850 state.driver_started_at = None;
16851 state.begin_pass();
16852 state.save().unwrap();
16853 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
16854 std::fs::write(
16855 state.dir().join("driver.json"),
16856 serde_json::to_vec(&serde_json::json!({
16857 "pid": state.driver_pid, "started_at": null,
16858 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
16859 }))
16860 .unwrap(),
16861 )
16862 .unwrap();
16863 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
16864 drop(lease);
16865 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
16866 let mut resumed = Runner::resume(&state.id).unwrap();
16867 assert!(!resumed.state.driver_exited);
16868 let mut execution = resumed
16869 .claim_driver()
16870 .await
16871 .expect("lease proves previous execution ended");
16872 assert_eq!(execution.runner.state.passes.len(), 2);
16873 execution
16874 .driver
16875 .finish(&mut execution.runner.state)
16876 .await
16877 .unwrap();
16878 }
16879
16880 #[tokio::test]
16881 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16882 crate::run::pin_test_home();
16883 let mut runner = runner_at(RunStatus::Reviewing);
16884 let mut json = serde_json::to_value(&runner.state).unwrap();
16885 json.as_object_mut().unwrap().remove("driver_claimed");
16886 runner.state = serde_json::from_value(json).unwrap();
16887 assert_eq!(runner.state.driver_claimed, None);
16888 runner.state.save().unwrap();
16889 let mut execution = runner
16890 .claim_driver()
16891 .await
16892 .expect("the execution lease, not a missing marker, excludes a second driver");
16893 assert_eq!(execution.runner.state.passes.len(), 1);
16894 execution
16895 .driver
16896 .finish(&mut execution.runner.state)
16897 .await
16898 .unwrap();
16899 }
16900
16901 #[tokio::test]
16902 async fn driver_admission_refuses_a_proven_live_driver() {
16903 crate::run::pin_test_home();
16904 let mut runner = runner_at(RunStatus::Reviewing);
16905 let pid = std::process::id();
16906 runner.state.driver_pid = Some(pid);
16907 runner.state.driver_started_at = crate::proc::process_started_at(pid);
16908 assert!(runner.state.driver_started_at.is_some());
16909 runner.state.save().unwrap();
16910 let error = runner
16911 .claim_driver()
16912 .await
16913 .err()
16914 .expect("a live driver must refuse");
16915 assert!(error.to_string().contains("live driver"), "{error:#}");
16916 let saved = RunState::load(&runner.state.id).unwrap();
16917 assert_eq!(saved.driver_pid, Some(pid));
16918 assert_eq!(saved.status, RunStatus::Reviewing);
16919 assert!(saved.passes.is_empty());
16920 }
16921
16922 #[test]
16923 fn parking_resource_halt_persists_the_pass_once() {
16924 crate::run::pin_test_home();
16925 let mut runner = runner_at(RunStatus::Reviewing);
16926 runner.state.begin_pass();
16927 runner.state.resource_block = Some(crate::run::ResourceBlock {
16928 node: "fix".into(),
16929 kind: "shared build cache".into(),
16930 detail: "cancelled".into(),
16931 at: Timestamp::now(),
16932 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16933 });
16934 let cause = ParkCause::Upgrade {
16935 from: "old".into(),
16936 to: Some("new".into()),
16937 };
16938 runner.pause.park_for(cause.clone());
16939 runner.stop_for_resource().unwrap();
16942 assert!(runner.resource_halt().unwrap());
16943 assert!(runner.park_here().unwrap());
16944 let saved = RunState::load(&runner.state.id).unwrap();
16945 assert!(saved.parked);
16946 assert_eq!(saved.status, RunStatus::Reviewing);
16947 assert!(saved.passes[0].ended_at.is_some());
16948 assert_eq!(
16949 saved.passes[0].end,
16950 Some(crate::run::PassEnd::Parked {
16951 cause: cause.clone()
16952 })
16953 );
16954 assert_eq!(
16955 saved.resource_block.unwrap().cause,
16956 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16957 );
16958 assert_eq!(
16959 saved
16960 .events
16961 .iter()
16962 .filter(|event| event.node == "park")
16963 .count(),
16964 1
16965 );
16966 }
16967
16968 #[test]
16976 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16977 crate::run::pin_test_home();
16978 let mut runner = runner_at(RunStatus::Implementing);
16979 let shutdown = Pause::new();
16980 runner.on_pause(shutdown.clone());
16981 let interrupt = Pause::new();
16982 runner.watch_interrupt(interrupt.clone());
16983
16984 assert!(!runner.park_here().expect("park_here"));
16986 assert!(!runner.state.parked);
16987
16988 interrupt.park_because("test");
16990 assert!(!shutdown.parked());
16991 assert!(runner.park_here().expect("park_here"));
16992 }
16993
16994 #[tokio::test]
17008 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
17009 crate::run::pin_test_home();
17010 let mut runner = runner_at(RunStatus::Implementing);
17011 let interrupt = Pause::new();
17012 runner.watch_interrupt(interrupt.clone());
17013
17014 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
17015 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
17016
17017 let node = async move {
17021 started_tx.send(()).expect("send started");
17022 finish_rx.await.expect("recv finish");
17023 "node finished"
17024 };
17025
17026 let interrupter = async move {
17027 started_rx.await.expect("recv started");
17028 interrupt.park_because("higher-priority task waiting");
17030 tokio::task::yield_now().await;
17034 finish_tx.send(()).expect("send finish");
17035 };
17036
17037 let (node_result, ()) = tokio::join!(node, interrupter);
17038 assert_eq!(
17039 node_result, "node finished",
17040 "the in-flight call ran to completion"
17041 );
17042
17043 assert!(runner.park_here().expect("park_here"));
17046 assert!(runner.state.parked);
17047 }
17048
17049 #[test]
17055 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
17056 crate::run::pin_test_home();
17057 let mut runner = runner_at(RunStatus::Judging);
17058 runner.state.config.agents = vec![conductor()];
17062 runner.state.candidates = vec![Candidate {
17063 index: 0,
17064 label: 'A',
17065 agent: "alpha".to_owned(),
17066 branch: "magi/x/A".to_owned(),
17067 worktree: PathBuf::from("/nonexistent/worktree"),
17068 summary: "did the thing".to_owned(),
17069 stat: "1 file changed".to_owned(),
17070 files: 1,
17071 commits: 1,
17072 empty: false,
17073 failed: None,
17074 verified_noop: None,
17075 duration_ms: 1234,
17076 folded: false,
17077 }];
17078 let run_id = runner.state.id.clone();
17079
17080 let interrupt = Pause::new();
17081 runner.watch_interrupt(interrupt.clone());
17082 interrupt.park_because("task c3d4 asked to run first");
17083 assert!(runner.park_here().expect("park_here"));
17084
17085 let resumed = Runner::resume(&run_id).expect("resume");
17086 assert_eq!(resumed.state.candidates.len(), 1);
17087 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
17088 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
17089 assert_eq!(resumed.state.status, runner.state.status);
17090 assert!(
17091 resumed.state.parked,
17092 "still parked until `execute` actually walks the graph again"
17093 );
17094 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
17095 }
17096
17097 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
17099 let mut q = ask::Question::new(
17100 run.to_owned(),
17101 "implement".to_owned(),
17102 "impl-A".to_owned(),
17103 "Which storage backend should the cache use?".to_owned(),
17104 String::new(),
17105 vec!["SQLite".to_owned(), "Redis".to_owned()],
17106 );
17107 store.put(&mut q).unwrap();
17108 q
17109 }
17110
17111 #[test]
17112 fn a_failed_runs_open_question_is_abandoned() {
17113 ask_test_home();
17114 let store = ask::Questions::open();
17115 let mut runner = runner_at(RunStatus::Failed);
17116 let run = runner.state.id.clone();
17117 let q = ask_open_question(&store, &run);
17118
17119 runner.settle_questions();
17120
17121 let back = store.get(&q.id).unwrap();
17122 assert!(
17123 !back.status.open(),
17124 "the seat that asked died with the run; nobody is left to read an answer"
17125 );
17126 assert!(
17127 back.detail.contains(&run) && back.detail.contains("failed"),
17128 "the reason names what the run became, not just that it is gone: {}",
17129 back.detail
17130 );
17131 }
17132
17133 #[test]
17134 fn a_merged_runs_open_question_is_abandoned_too() {
17135 ask_test_home();
17136 let store = ask::Questions::open();
17137 for status in [RunStatus::Merged, RunStatus::Ready] {
17140 let mut runner = runner_at(status);
17141 let run = runner.state.id.clone();
17142 let q = ask_open_question(&store, &run);
17143
17144 runner.settle_questions();
17145
17146 let back = store.get(&q.id).unwrap();
17147 assert!(
17148 !back.status.open(),
17149 "{status:?} run's question must not outlive the run"
17150 );
17151 }
17152 }
17153
17154 #[test]
17155 fn a_still_resumable_runs_open_question_is_left_alone() {
17156 ask_test_home();
17157 let store = ask::Questions::open();
17158 for status in [RunStatus::Blocked, RunStatus::Stalled] {
17164 let mut runner = runner_at(status);
17165 let run = runner.state.id.clone();
17166 let q = ask_open_question(&store, &run);
17167
17168 runner.settle_questions();
17169
17170 let back = store.get(&q.id).unwrap();
17171 assert!(
17172 back.status.open(),
17173 "{status:?} is still alive; the question must still be waiting"
17174 );
17175 }
17176 }
17177
17178 #[test]
17179 fn settle_questions_never_touches_an_already_answered_question() {
17180 ask_test_home();
17181 let store = ask::Questions::open();
17182 let mut runner = runner_at(RunStatus::Failed);
17183 let run = runner.state.id.clone();
17184 let mut q = ask_open_question(&store, &run);
17185 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
17186 .unwrap();
17187 store.put(&mut q).unwrap();
17188
17189 runner.settle_questions();
17194 runner.settle_questions();
17195
17196 let back = store.get(&q.id).unwrap();
17197 assert_eq!(
17198 back.status,
17199 ask::QuestionStatus::Answered,
17200 "a real answer is a decision on record, never overwritten by a sweep"
17201 );
17202 }
17203
17204 #[tokio::test]
17215 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
17216 crate::run::pin_test_home();
17217 let tmp = tempfile::tempdir().expect("tempdir");
17218 let repo = tmp.path().join("repo");
17219 std::fs::create_dir_all(&repo).unwrap();
17220 init_repo(&repo);
17221
17222 let mut config = Config::default();
17223 config.graph.worktree_root = Some(tmp.path().join("wt"));
17224
17225 let mut state = RunState::new(
17226 repo.clone(),
17227 "main".to_owned(),
17228 "deadbeef".to_owned(),
17229 "task".to_owned(),
17230 config,
17231 );
17232 let root = state.worktree_root();
17233 let wt_a = root.join("cand-A");
17234 let wt_b = root.join("cand-B");
17235 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
17236 .await
17237 .expect("worktree A");
17238 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
17239 .await
17240 .expect("worktree B");
17241
17242 state.candidates = vec![
17243 Candidate {
17244 index: 0,
17245 label: 'A',
17246 agent: "alpha".to_owned(),
17247 branch: "magi/x/A".to_owned(),
17248 worktree: wt_a.clone(),
17249 summary: String::new(),
17250 stat: String::new(),
17251 files: 0,
17252 commits: 0,
17253 empty: false,
17254 failed: None,
17255 verified_noop: None,
17256 duration_ms: 0,
17257 folded: false,
17258 },
17259 Candidate {
17260 index: 1,
17261 label: 'B',
17262 agent: "beta".to_owned(),
17263 branch: "magi/x/B".to_owned(),
17264 worktree: wt_b.clone(),
17265 summary: String::new(),
17266 stat: String::new(),
17267 files: 0,
17268 commits: 0,
17269 empty: false,
17270 failed: None,
17271 verified_noop: None,
17272 duration_ms: 0,
17273 folded: false,
17274 },
17275 ];
17276 state.tally = Some(Tally {
17277 first_choice: BTreeMap::from([('A', 1)]),
17278 borda: BTreeMap::new(),
17279 winner: 'A',
17280 rankings: 1,
17281 unanimous_initial: true,
17282 deliberated: false,
17283 changed_votes: 0,
17284 unanimous_final: true,
17285 tie_break: None,
17286 judges: 1,
17287 present: 1,
17288 quorum: 1,
17289 met_quorum: true,
17290 uncontested: None,
17291 });
17292 state.status = RunStatus::Ready;
17293
17294 fold_run(&mut state, false, &crate::run::home())
17295 .await
17296 .expect("fold_run");
17297
17298 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17299 assert!(
17300 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17301 "the unmerged winner's branch survives"
17302 );
17303 assert!(
17304 !state.candidates[0].folded,
17305 "the winner is not marked folded"
17306 );
17307
17308 assert!(!wt_b.exists(), "the loser's worktree is removed");
17309 assert!(
17310 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17311 "the loser's branch is removed"
17312 );
17313 assert!(state.candidates[1].folded, "the loser is marked folded");
17314 }
17315
17316 #[tokio::test]
17317 #[cfg(unix)]
17318 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17319 use std::os::fd::AsRawFd as _;
17320 use std::time::Duration;
17321
17322 let tmp = tempfile::tempdir().unwrap();
17323 let repo = tmp.path().join("repo");
17324 std::fs::create_dir_all(&repo).unwrap();
17325 init_repo(&repo);
17326 let common = repo.join(".git");
17327 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17328 let home = tmp.path().join("home");
17329 let mut config = Config::default();
17330 config.coordination = crate::config::test_coordination();
17331 config.graph.worktree_root = Some(tmp.path().join("wt"));
17332 config.graph.timeout_resource_wait = 1;
17333 let mut state = RunState::new(
17334 repo.clone(),
17335 "main".into(),
17336 "deadbeef".into(),
17337 "task".into(),
17338 config,
17339 );
17340 let root = state.worktree_root();
17341 let candidate = root.join("cand-A");
17342 let reviewer = root.join("review-1");
17343 let branch = state.branch_for('A');
17344 git::worktree_add_branch(&repo, &candidate, &branch, "main")
17345 .await
17346 .unwrap();
17347 git::worktree_add_detached(&repo, &reviewer, "main")
17348 .await
17349 .unwrap();
17350 git::acquire_worktree_config(&repo).await.unwrap();
17351 state.enabled_worktree_config = true;
17352 state.status = RunStatus::Ready;
17353 state.candidates.push(Candidate {
17354 index: 0,
17355 label: 'A',
17356 agent: "implement".into(),
17357 branch: branch.clone(),
17358 worktree: candidate.clone(),
17359 summary: String::new(),
17360 stat: String::new(),
17361 files: 0,
17362 commits: 0,
17363 empty: false,
17364 failed: None,
17365 verified_noop: None,
17366 duration_ms: 0,
17367 folded: false,
17368 });
17369 state.save_under(&home).unwrap();
17370 let config_before = std::fs::read(common.join("config")).unwrap();
17371 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17372 .await
17373 .unwrap();
17374 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17377 assert_eq!(
17378 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17379 0
17380 );
17381 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17382 assert!(
17383 tokio::time::timeout(Duration::from_millis(200), &mut fold)
17384 .await
17385 .is_err()
17386 );
17387 assert!(candidate.exists());
17388 assert!(reviewer.exists());
17389 assert!(git::branch_exists(&repo, &branch).await.unwrap());
17390 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17391 assert_eq!(
17392 git::git(&repo, &["worktree", "list", "--porcelain"])
17393 .await
17394 .unwrap(),
17395 worktrees_before
17396 );
17397 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17398 .await
17399 .unwrap()
17400 .unwrap_err();
17401 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17402 drop(fold);
17403 assert!(candidate.exists() && reviewer.exists());
17404 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17405 assert_eq!(
17406 state.resource_block.as_ref().unwrap().kind,
17407 "repository coordination"
17408 );
17409 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17410 drop(external);
17411 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17412 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17413 .await
17414 .unwrap()
17415 .unwrap();
17416 drop(fold);
17417 assert!(removed.contains(&branch));
17418 assert!(!root.exists());
17419 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17420 assert!(state.candidates[0].folded);
17421 assert!(!state.enabled_worktree_config);
17422 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17423 let enabled = git::git_raw(
17424 &repo,
17425 &["config", "--local", "--get", "extensions.worktreeConfig"],
17426 )
17427 .await
17428 .unwrap();
17429 assert!(!enabled.ok(), "the last config reference was released");
17430 }
17431
17432 #[tokio::test]
17435 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17436 let tmp = tempfile::tempdir().expect("tempdir");
17437 let repo = tmp.path().join("repo");
17438 std::fs::create_dir_all(&repo).unwrap();
17439 init_repo(&repo);
17440 let home = tmp.path().join("home");
17441
17442 let mut config = Config::default();
17443 config.graph.worktree_root = Some(tmp.path().join("wt"));
17444 let mut state = RunState::new(
17445 repo.clone(),
17446 "main".to_owned(),
17447 "deadbeef".to_owned(),
17448 "task".to_owned(),
17449 config,
17450 );
17451 git::git(&repo, &["branch", "magi/x/A", "main"])
17453 .await
17454 .expect("branch");
17455 state.candidates = vec![Candidate {
17456 index: 0,
17457 label: 'A',
17458 agent: "alpha".to_owned(),
17459 branch: "magi/x/A".to_owned(),
17460 worktree: state.worktree_root().join("cand-A"),
17461 summary: String::new(),
17462 stat: String::new(),
17463 files: 0,
17464 commits: 0,
17465 empty: false,
17466 failed: None,
17467 verified_noop: None,
17468 duration_ms: 0,
17469 folded: true,
17470 }];
17471 state.released_to = Some("20260901-000000-new1".to_owned());
17472 state.released_branches = vec!["magi/x/A".to_owned()];
17473
17474 fold_run(&mut state, true, &home).await.expect("fold_run");
17475 state.status = RunStatus::Merged;
17476 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17477 state.merge = Some(crate::run::MergeOutcome {
17478 mode: MergeMode::Pr,
17479 ok: true,
17480 detail: "confirmed merged".into(),
17481 empty: false,
17482 });
17483 fold_run(&mut state, true, &home)
17484 .await
17485 .expect("merged fold");
17486
17487 assert!(
17488 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17489 "the handed-over branch survives a fold"
17490 );
17491 }
17492
17493 #[tokio::test]
17497 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17498 let tmp = tempfile::tempdir().expect("tempdir");
17499 let repo = tmp.path().join("repo");
17500 std::fs::create_dir_all(&repo).unwrap();
17501 init_repo(&repo);
17502 let run = |args: &[&str]| {
17503 let out = std::process::Command::new("git")
17504 .quiet()
17505 .args(args)
17506 .current_dir(&repo)
17507 .output()
17508 .expect("spawn git");
17509 assert!(out.status.success(), "git {args:?}");
17510 };
17511 run(&["branch", "magi/x/A"]);
17512 run(&["checkout", "-q", "-b", "magi/x/B"]);
17513 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17514 run(&["add", "-A"]);
17515 run(&["commit", "-q", "-m", "work"]);
17516 run(&["checkout", "-q", "main"]);
17517
17518 let probe = RunState::new(
17522 repo.clone(),
17523 "main".to_owned(),
17524 "deadbeef".to_owned(),
17525 "task".to_owned(),
17526 Config::default(),
17527 );
17528 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17529 let bare = tmp.path().join("origin.git");
17530 let out = std::process::Command::new("git")
17531 .quiet()
17532 .args(["init", "-q", "--bare"])
17533 .arg(&bare)
17534 .output()
17535 .expect("spawn git");
17536 assert!(out.status.success(), "git init --bare");
17537 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17538 run(&["push", "-q", "origin", "main"]);
17539
17540 let mut state = RunState::new(
17541 repo.clone(),
17542 "main".to_owned(),
17543 "deadbeef".to_owned(),
17544 "task".to_owned(),
17545 Config::default(),
17546 );
17547 state.seeds = vec![refs::Seed {
17548 token: "magi/27b2/A".to_owned(),
17549 kind: refs::SeedKind::Unresolved,
17550 sha: String::new(),
17551 branch: true,
17552 detail: "no branch or commit named magi/27b2/A".to_owned(),
17553 }];
17554
17555 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17556 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17557 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17558 let detail = empty_candidate_detail(&state, "main");
17559 assert!(detail.starts_with("empty candidate"), "{detail}");
17560 assert!(!detail.contains("magi/27b2/A"), "{detail}");
17561 assert!(detail.contains("--start-from"), "{detail}");
17562 }
17563
17564 #[tokio::test]
17573 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17574 let tmp = tempfile::tempdir().expect("tempdir");
17575 let repo = tmp.path().join("repo");
17576 std::fs::create_dir_all(&repo).unwrap();
17577 init_repo(&repo);
17578
17579 let mut config = Config::default();
17580 config.merge.mode = MergeMode::Local;
17581
17582 let mut state = RunState::new(
17583 repo.clone(),
17584 "main".to_owned(),
17585 "deadbeef".to_owned(),
17586 "task".to_owned(),
17587 config,
17588 );
17589 state.candidates = vec![Candidate {
17590 index: 0,
17591 label: 'A',
17592 agent: "alpha".to_owned(),
17593 branch: "does-not-exist".to_owned(),
17594 worktree: repo.clone(),
17595 summary: String::new(),
17596 stat: String::new(),
17597 files: 0,
17598 commits: 0,
17599 empty: false,
17600 failed: None,
17601 verified_noop: None,
17602 duration_ms: 0,
17603 folded: false,
17604 }];
17605 state.tally = Some(Tally {
17606 first_choice: BTreeMap::from([('A', 1)]),
17607 borda: BTreeMap::new(),
17608 winner: 'A',
17609 rankings: 1,
17610 unanimous_initial: true,
17611 deliberated: false,
17612 changed_votes: 0,
17613 unanimous_final: true,
17614 tie_break: None,
17615 judges: 0,
17616 present: 0,
17617 quorum: 0,
17618 met_quorum: true,
17619 uncontested: Some("only candidate A produced a change".to_owned()),
17620 });
17621 state.reviews = vec![ReviewRound {
17622 round: 1,
17623 head: "deadbeef".to_owned(),
17624 verified_head: None,
17625 verified_at: None,
17626 reviews: Vec::new(),
17627 e2e: Vec::new(),
17628 fix: None,
17629 blocking: 0,
17630 answered: 0,
17631 expected: 0,
17632 clean: true,
17633 verify_retried: false,
17634 e2e_deferred: false,
17635 e2e_defer_reason: None,
17636 progressed: false,
17637 vote_split: false,
17638 reconsideration: Vec::new(),
17639 verdict: None,
17640 }];
17641 state.gate = vec![CommandOutcome {
17642 command: "test".to_owned(),
17643 code: Some(0),
17644 output_tail: String::new(),
17645 duration_ms: 0,
17646 resource_blocked: false,
17647 }];
17648 state.gate_ran = true;
17649 state.status = RunStatus::Ready;
17654 state.merge = Some(MergeOutcome {
17655 mode: MergeMode::Local,
17656 ok: false,
17657 detail: "already concluded".to_owned(),
17658 empty: false,
17659 });
17660
17661 let mut runner = Runner {
17662 state,
17663 roles: ResolvedRoles {
17664 implementers: Vec::new(),
17665 judges: Vec::new(),
17666 reviewers: Vec::new(),
17667 fixer: None,
17668 conductor: conductor(),
17669 implementer_roster: Vec::new(),
17670 judge_roster: Vec::new(),
17671 reviewer_roster: Vec::new(),
17672 },
17673 sem: Arc::new(Semaphore::new(1)),
17674 pause: Pause::new(),
17675 interrupt: Pause::new(),
17676 };
17677
17678 runner.merge().await.expect("merge");
17679
17680 assert_eq!(
17681 runner.state.status,
17682 RunStatus::Ready,
17683 "a concluded run's status must not change on reentry"
17684 );
17685 assert_eq!(
17686 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17687 Some("already concluded"),
17688 "merge must not run again once the node already recorded an outcome"
17689 );
17690 }
17691
17692 #[tokio::test]
17701 async fn merge_refuses_a_gate_that_has_not_actually_run() {
17702 let tmp = tempfile::tempdir().expect("tempdir");
17703 let repo = tmp.path().join("repo");
17704 std::fs::create_dir_all(&repo).unwrap();
17705 init_repo(&repo);
17706
17707 let mut config = Config::default();
17708 config.merge.mode = MergeMode::Local;
17709
17710 let mut state = RunState::new(
17711 repo.clone(),
17712 "main".to_owned(),
17713 "deadbeef".to_owned(),
17714 "task".to_owned(),
17715 config,
17716 );
17717 state.candidates = vec![Candidate {
17718 index: 0,
17719 label: 'A',
17720 agent: "alpha".to_owned(),
17721 branch: "does-not-exist".to_owned(),
17722 worktree: repo.clone(),
17723 summary: String::new(),
17724 stat: String::new(),
17725 files: 0,
17726 commits: 0,
17727 empty: false,
17728 failed: None,
17729 verified_noop: None,
17730 duration_ms: 0,
17731 folded: false,
17732 }];
17733 state.tally = Some(Tally {
17734 first_choice: BTreeMap::from([('A', 1)]),
17735 borda: BTreeMap::new(),
17736 winner: 'A',
17737 rankings: 1,
17738 unanimous_initial: true,
17739 deliberated: false,
17740 changed_votes: 0,
17741 unanimous_final: true,
17742 tie_break: None,
17743 judges: 0,
17744 present: 0,
17745 quorum: 0,
17746 met_quorum: true,
17747 uncontested: Some("only candidate A produced a change".to_owned()),
17748 });
17749 state.reviews = vec![ReviewRound {
17750 round: 1,
17751 head: "deadbeef".to_owned(),
17752 verified_head: None,
17753 verified_at: None,
17754 reviews: Vec::new(),
17755 e2e: Vec::new(),
17756 fix: None,
17757 blocking: 0,
17758 answered: 0,
17759 expected: 0,
17760 clean: true,
17761 verify_retried: false,
17762 e2e_deferred: false,
17763 e2e_defer_reason: None,
17764 progressed: false,
17765 vote_split: false,
17766 reconsideration: Vec::new(),
17767 verdict: None,
17768 }];
17769 state.gate = Vec::new();
17771 state.gate_ran = false;
17772 state.status = RunStatus::Gating;
17773
17774 let mut runner = Runner {
17775 state,
17776 roles: ResolvedRoles {
17777 implementers: Vec::new(),
17778 judges: Vec::new(),
17779 reviewers: Vec::new(),
17780 fixer: None,
17781 conductor: conductor(),
17782 implementer_roster: Vec::new(),
17783 judge_roster: Vec::new(),
17784 reviewer_roster: Vec::new(),
17785 },
17786 sem: Arc::new(Semaphore::new(1)),
17787 pause: Pause::new(),
17788 interrupt: Pause::new(),
17789 };
17790
17791 runner.merge().await.expect("merge");
17792
17793 assert!(
17794 runner.state.merge.is_none(),
17795 "an empty gate must never be read as a passing one: {:?}",
17796 runner.state.merge
17797 );
17798 }
17799
17800 #[tokio::test]
17807 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
17808 let tmp = tempfile::tempdir().expect("tempdir");
17809 let repo = tmp.path().join("repo");
17810 std::fs::create_dir_all(&repo).unwrap();
17811 init_repo(&repo);
17812
17813 let config = Config::default();
17815
17816 let mut state = RunState::new(
17817 repo.clone(),
17818 "main".to_owned(),
17819 "deadbeef".to_owned(),
17820 "task".to_owned(),
17821 config,
17822 );
17823 state.candidates = vec![Candidate {
17824 index: 0,
17825 label: 'A',
17826 agent: "alpha".to_owned(),
17827 branch: "does-not-exist".to_owned(),
17828 worktree: repo.clone(),
17829 summary: String::new(),
17830 stat: String::new(),
17831 files: 0,
17832 commits: 0,
17833 empty: false,
17834 failed: None,
17835 verified_noop: None,
17836 duration_ms: 0,
17837 folded: false,
17838 }];
17839 state.tally = Some(Tally {
17840 first_choice: BTreeMap::from([('A', 1)]),
17841 borda: BTreeMap::new(),
17842 winner: 'A',
17843 rankings: 1,
17844 unanimous_initial: true,
17845 deliberated: false,
17846 changed_votes: 0,
17847 unanimous_final: true,
17848 tie_break: None,
17849 judges: 0,
17850 present: 0,
17851 quorum: 0,
17852 met_quorum: true,
17853 uncontested: Some("only candidate A produced a change".to_owned()),
17854 });
17855 state.reviews = vec![ReviewRound {
17856 round: 1,
17857 head: "deadbeef".to_owned(),
17858 verified_head: None,
17859 verified_at: None,
17860 reviews: Vec::new(),
17861 e2e: Vec::new(),
17862 fix: None,
17863 blocking: 0,
17864 answered: 0,
17865 expected: 0,
17866 clean: true,
17867 verify_retried: false,
17868 e2e_deferred: false,
17869 e2e_defer_reason: None,
17870 progressed: false,
17871 vote_split: false,
17872 reconsideration: Vec::new(),
17873 verdict: None,
17874 }];
17875
17876 let mut runner = Runner {
17877 state,
17878 roles: ResolvedRoles {
17879 implementers: Vec::new(),
17880 judges: Vec::new(),
17881 reviewers: Vec::new(),
17882 fixer: None,
17883 conductor: conductor(),
17884 implementer_roster: Vec::new(),
17885 judge_roster: Vec::new(),
17886 reviewer_roster: Vec::new(),
17887 },
17888 sem: Arc::new(Semaphore::new(1)),
17889 pause: Pause::new(),
17890 interrupt: Pause::new(),
17891 };
17892
17893 runner.gate().await.expect("gate");
17894 assert!(
17895 runner.state.gate_ran,
17896 "zero configured commands is still a real attempt, not an unrun gate"
17897 );
17898 assert!(runner.state.gate.is_empty());
17899 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17900 assert_ne!(
17901 runner.state.status,
17902 RunStatus::Blocked,
17903 "a gate with nothing to check must not read as failed"
17904 );
17905
17906 runner.merge().await.expect("merge");
17907 assert_eq!(
17908 runner.state.status,
17909 RunStatus::Ready,
17910 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17911 );
17912 }
17913
17914 #[tokio::test]
17925 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17926 crate::run::pin_test_home();
17927 let home = crate::run::home();
17928
17929 let tmp = tempfile::tempdir().expect("tempdir");
17930 let repo = tmp.path().join("repo");
17931 std::fs::create_dir_all(&repo).unwrap();
17932 init_repo(&repo);
17933 let cache_dir = tmp.path().join("target");
17936
17937 let mut config = Config::default();
17938 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17939 config.graph.timeout_verify = Some(2);
17942 config.graph.timeout_resource_wait = 2;
17943
17944 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17945 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17946 .expect("no io error acquiring directly")
17947 {
17948 crate::cache::AcquireOutcome::Acquired(g) => g,
17949 crate::cache::AcquireOutcome::Busy(b) => {
17950 panic!("expected the direct acquire to win the lease first: {b:?}")
17951 }
17952 };
17953
17954 let mut state = RunState::new(
17955 repo.clone(),
17956 "main".to_owned(),
17957 "deadbeef".to_owned(),
17958 "task".to_owned(),
17959 config,
17960 );
17961 state.candidates = vec![Candidate {
17962 index: 0,
17963 label: 'A',
17964 agent: "alpha".to_owned(),
17965 branch: "does-not-exist".to_owned(),
17966 worktree: repo.clone(),
17967 summary: String::new(),
17968 stat: String::new(),
17969 files: 0,
17970 commits: 0,
17971 empty: false,
17972 failed: None,
17973 verified_noop: None,
17974 duration_ms: 0,
17975 folded: false,
17976 }];
17977 state.tally = Some(Tally {
17978 first_choice: BTreeMap::from([('A', 1)]),
17979 borda: BTreeMap::new(),
17980 winner: 'A',
17981 rankings: 1,
17982 unanimous_initial: true,
17983 deliberated: false,
17984 changed_votes: 0,
17985 unanimous_final: true,
17986 tie_break: None,
17987 judges: 0,
17988 present: 0,
17989 quorum: 0,
17990 met_quorum: true,
17991 uncontested: Some("only candidate A produced a change".to_owned()),
17992 });
17993 state.reviews = vec![ReviewRound {
17994 round: 1,
17995 head: "deadbeef".to_owned(),
17996 verified_head: None,
17997 verified_at: None,
17998 reviews: Vec::new(),
17999 e2e: Vec::new(),
18000 fix: None,
18001 blocking: 0,
18002 answered: 0,
18003 expected: 0,
18004 clean: true,
18005 verify_retried: false,
18006 e2e_deferred: false,
18007 e2e_defer_reason: None,
18008 progressed: false,
18009 vote_split: false,
18010 reconsideration: Vec::new(),
18011 verdict: None,
18012 }];
18013
18014 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
18015 state.reviews[0].e2e = vec![CommandOutcome {
18016 command: state.config.verify.e2e[0].clone(),
18017 code: Some(0),
18018 output_tail: String::new(),
18019 duration_ms: 0,
18020 resource_blocked: false,
18021 }];
18022 let mut runner = Runner {
18023 state,
18024 roles: ResolvedRoles {
18025 implementers: Vec::new(),
18026 judges: Vec::new(),
18027 reviewers: Vec::new(),
18028 fixer: None,
18029 conductor: conductor(),
18030 implementer_roster: Vec::new(),
18031 judge_roster: Vec::new(),
18032 reviewer_roster: Vec::new(),
18033 },
18034 sem: Arc::new(Semaphore::new(1)),
18035 pause: Pause::new(),
18036 interrupt: Pause::new(),
18037 };
18038
18039 let started = std::time::Instant::now();
18040 runner.gate().await.expect("gate");
18041 assert!(
18042 started.elapsed() < Duration::from_secs(1),
18043 "a gate with nothing to run must never wait on a lease it never needed"
18044 );
18045 assert!(
18046 runner.state.gate_ran,
18047 "zero commands is still a real, immediate attempt"
18048 );
18049 assert!(runner.state.gate.is_empty());
18050 assert_ne!(
18051 runner.state.status,
18052 RunStatus::Blocked,
18053 "must not read as resource-blocked on a lease it never asked for"
18054 );
18055 }
18056
18057 #[cfg(unix)]
18060 #[tokio::test]
18061 async fn verification_is_standalone_inside_a_live_worker() {
18062 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
18063 if std::env::var_os(CHILD).is_none() {
18064 let home = tempfile::tempdir().unwrap();
18065 let generation = crate::worker::pin_current(home.path()).unwrap();
18066 let output = std::process::Command::new(std::env::current_exe().unwrap())
18067 .args([
18068 "--exact",
18069 "graph::tests::verification_is_standalone_inside_a_live_worker",
18070 "--nocapture",
18071 ])
18072 .env(CHILD, "1")
18073 .env("MAGI_HOME", home.path())
18074 .env("MAGI_WORKER_HOME", home.path())
18075 .env("MAGI_WORKER_ID", "abcdef")
18076 .env("MAGI_WORKER_GENERATION", &generation.id)
18077 .env("MAGI_WORKER_EXE", &generation.executable)
18078 .env("MAGI_RUN", "verification-parent")
18079 .env("MAGI_NODE", "gate")
18080 .env("MAGI_SEAT", "worker-fixture")
18081 .env("MAGI_PROCESS_OWNER", "fixture-owner")
18082 .env("MAGI_NO_AUTOUPDATE", "0")
18083 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
18084 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
18085 .env(
18086 "VERIFY_EXPECT_TOOLCHAIN",
18087 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
18088 )
18089 .env(
18090 "VERIFY_EXPECT_CACHE",
18091 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
18092 )
18093 .output()
18094 .unwrap();
18095 assert!(
18096 output.status.success()
18097 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
18098 "{}\n{}",
18099 String::from_utf8_lossy(&output.stdout),
18100 String::from_utf8_lossy(&output.stderr)
18101 );
18102 return;
18103 }
18104 let home = crate::run::home();
18105 let directory = home.join("workers/jobs/abcdef");
18106 std::fs::create_dir_all(&directory).unwrap();
18107 let job = crate::worker::Job {
18108 id: "abcdef".into(),
18109 sequence: 1,
18110 kind: "task".into(),
18111 key: "task:verification".into(),
18112 payload: serde_json::json!({}),
18113 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
18114 version: Some(env!("CARGO_PKG_VERSION").into()),
18115 status: crate::worker::Status::Running,
18116 result: None,
18117 pid: Some(std::process::id()),
18118 created_at: 0,
18119 finished_at: None,
18120 };
18121 std::fs::write(
18122 directory.join("job.json"),
18123 serde_json::to_vec(&job).unwrap(),
18124 )
18125 .unwrap();
18126 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
18127 let assert_worker = || {
18128 assert_eq!(
18129 crate::worker::current_job(&home).unwrap().unwrap().id,
18130 job.id
18131 );
18132 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
18133 assert_eq!(
18134 crate::worker::helper_executable(&home).unwrap(),
18135 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
18136 );
18137 assert!(
18138 crate::updater::Checker::new(&crate::config::Update {
18139 mode: crate::config::UpdateMode::Notify,
18140 interval: None,
18141 auto_restart: false,
18142 })
18143 .is_none(),
18144 "the driver remains pinned, even while tests are standalone"
18145 );
18146 };
18147 assert_worker();
18148 let repo = home.join("repo");
18149 std::fs::create_dir_all(&repo).unwrap();
18150 init_repo(&repo);
18151 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
18154 test -z "${MAGI_WORKER_GENERATION+x}" &&
18155 test -z "${MAGI_WORKER_HOME+x}" &&
18156 test -z "${MAGI_WORKER_EXE+x}" &&
18157 test "$MAGI_HOME/repo" -ef . &&
18158 test "$MAGI_RUN" = verification-parent &&
18159 test "$MAGI_NODE" = gate &&
18160 test "$MAGI_SEAT" = worker-fixture &&
18161 test -n "$MAGI_PROCESS_OWNER" &&
18162 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
18163 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
18164 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
18165 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
18166 assert!(
18168 !std::process::Command::new("sh")
18169 .args(["-c", probe])
18170 .current_dir(&repo)
18171 .status()
18172 .unwrap()
18173 .success()
18174 );
18175 let unrelated_home = tempfile::tempdir().unwrap();
18176 assert!(
18177 crate::worker::current_job(unrelated_home.path()).is_err(),
18178 "an inherited worker id cannot be resolved in a unit fixture's home"
18179 );
18180 let mut runner = runner_at(RunStatus::Gating);
18181 runner.state.repo = repo.clone();
18182 runner.state.config.disk.min_free_bytes = 0;
18183 runner.state.config.coordination = crate::config::test_coordination();
18184 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
18185 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
18186 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
18187 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
18188 let winner = Candidate {
18189 index: 0,
18190 label: 'A',
18191 agent: "fixture".into(),
18192 branch: "main".into(),
18193 worktree: repo.clone(),
18194 summary: String::new(),
18195 stat: String::new(),
18196 files: 0,
18197 commits: 0,
18198 empty: false,
18199 failed: None,
18200 verified_noop: None,
18201 duration_ms: 0,
18202 folded: false,
18203 };
18204 let (e2e, retried, pids) = run_e2e_with_retry(
18205 &mut runner.state,
18206 &["sh".into(), "-c".into()],
18207 &[format!("{probe} && printf initial")],
18208 &repo,
18209 Duration::from_secs(10),
18210 "initial",
18211 )
18212 .await;
18213 assert!(!retried && pids.is_empty());
18214 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
18215 assert_eq!(e2e[0].output_tail, "initial");
18216 let gate = runner.run_gate(&winner).await.unwrap();
18217 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
18218 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
18221 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
18222 passed: false,
18223 outcomes: Vec::new(),
18224 });
18225 runner.state.save().unwrap();
18226 let mut resumed = RunState::load(&runner.state.id).unwrap();
18227 let wait = crate::repo_lock::WaitPolicy::default();
18228 assert!(matches!(
18229 check_gate_repair_e2e(&mut resumed, &winner, &wait)
18230 .await
18231 .unwrap(),
18232 GateFix::Retry
18233 ));
18234 let saved = RunState::load(&resumed.id).unwrap();
18235 let repair = saved.gate_repair_e2e.unwrap();
18236 assert!(repair.passed);
18237 assert_eq!(repair.outcomes.len(), 1);
18238 assert!(repair.outcomes[0].ok());
18239 let landing = run_landing_gate(&mut resumed, &winner, &wait)
18240 .await
18241 .unwrap();
18242 assert_eq!(landing.len(), 1);
18243 assert!(landing[0].ok(), "{landing:?}");
18244 assert_eq!(
18245 std::fs::read_to_string(repo.join("phases")).unwrap(),
18246 "pre\ngate\ne2e\npre\ngate\n"
18247 );
18248 assert_worker();
18249 }
18250
18251 #[cfg(unix)]
18252 #[tokio::test]
18253 async fn verification_keeps_the_panic_and_the_full_output() {
18254 crate::run::pin_test_home();
18255 let tmp = tempfile::tempdir().unwrap();
18256 let mut config = Config::default();
18257 config.disk.min_free_bytes = 0;
18258 let mut state = RunState::new(
18259 tmp.path().to_owned(),
18260 "main".into(),
18261 "base".into(),
18262 "verification evidence".into(),
18263 config,
18264 );
18265 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
18266 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
18267 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
18268 let mut names = Vec::new();
18269 for _ in 0..2 {
18270 let (outcomes, _) = run_commands(
18271 &mut state,
18272 "gate",
18273 "gate",
18274 0,
18275 &["sh".into(), "-c".into()],
18276 &[script.into()],
18277 tmp.path(),
18278 Duration::from_secs(60),
18279 )
18280 .await;
18281 let tail = &outcomes[0].output_tail;
18282 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18283 assert!(tail.contains("boom assertion"));
18284 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18285 let line = tail.lines().next().unwrap();
18286 names.push(line.to_owned());
18287 }
18288 assert_ne!(names[0], names[1], "the second run must not overwrite");
18289 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18290 assert!(full.len() > 500_000 && full.contains("boom assertion"));
18291 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18292
18293 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18295 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18296 let (outcomes, _) = run_commands(
18297 &mut state,
18298 "gate",
18299 "gate",
18300 0,
18301 &["sh".into(), "-c".into()],
18302 &["echo failure-output; exit 7".into()],
18303 tmp.path(),
18304 Duration::from_secs(10),
18305 )
18306 .await;
18307 assert_eq!(outcomes[0].code, Some(7));
18308 assert!(state.events.iter().any(|e| {
18309 e.node == "gate" && e.message.contains("could not save the full gate output:")
18310 }));
18311 }
18312
18313 #[cfg(unix)]
18314 #[tokio::test]
18315 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18316 crate::run::pin_test_home();
18317 let tmp = tempfile::tempdir().unwrap();
18318 let mut config = Config::default();
18319 config.disk.min_free_bytes = 0;
18320 let mut state = RunState::new(
18321 tmp.path().to_owned(),
18322 "main".into(),
18323 "base".into(),
18324 "verification evidence".into(),
18325 config,
18326 );
18327 let (outcomes, pids) = run_commands(
18328 &mut state,
18329 "gate",
18330 "gate",
18331 1,
18332 &["sh".into(), "-c".into()],
18333 &[
18334 "printf before-timeout; exec sleep 30".into(),
18335 "kill -TERM $$".into(),
18336 "exit 7".into(),
18337 "printf success".into(),
18338 ],
18339 tmp.path(),
18340 Duration::from_secs(1),
18341 )
18342 .await;
18343 wait_for_timed_out_children_to_die(&pids).await;
18344 assert_eq!(
18345 pids.len(),
18346 1,
18347 "unleased timeout retains its death-check PID"
18348 );
18349 assert_eq!(outcomes.len(), 4);
18350 assert_eq!(outcomes[0].code, None);
18351 assert!(outcomes[0].output_tail.contains("before-timeout"));
18352 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18353 assert_eq!(outcomes[1].code, None);
18354 assert!(outcomes[1].output_tail.contains("command terminated:"));
18355 assert!(!outcomes[1].output_tail.contains("timed out"));
18356 assert_eq!(outcomes[2].code, Some(7));
18357 assert!(!outcomes[2].output_tail.contains("timed out"));
18358 assert_eq!(outcomes[3].code, Some(0));
18359 assert_eq!(outcomes[3].output_tail, "success");
18360 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18361 }
18362
18363 #[tokio::test]
18373 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18374 crate::run::pin_test_home();
18375
18376 let tmp = tempfile::tempdir().expect("tempdir");
18377 let repo = tmp.path().join("repo");
18378 std::fs::create_dir_all(&repo).unwrap();
18379 init_repo(&repo);
18380
18381 let mut config = Config::default();
18382 config.verify.gate = vec![
18383 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18384 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18385 .to_owned(),
18386 ];
18387
18388 let mut state = RunState::new(
18389 repo.clone(),
18390 "main".to_owned(),
18391 "deadbeef".to_owned(),
18392 "task".to_owned(),
18393 config,
18394 );
18395 let run_id = state.id.clone();
18396 state.candidates = vec![Candidate {
18397 index: 0,
18398 label: 'A',
18399 agent: "alpha".to_owned(),
18400 branch: "does-not-exist".to_owned(),
18401 worktree: repo.clone(),
18402 summary: String::new(),
18403 stat: String::new(),
18404 files: 0,
18405 commits: 0,
18406 empty: false,
18407 failed: None,
18408 verified_noop: None,
18409 duration_ms: 0,
18410 folded: false,
18411 }];
18412 state.tally = Some(Tally {
18413 first_choice: BTreeMap::from([('A', 1)]),
18414 borda: BTreeMap::new(),
18415 winner: 'A',
18416 rankings: 1,
18417 unanimous_initial: true,
18418 deliberated: false,
18419 changed_votes: 0,
18420 unanimous_final: true,
18421 tie_break: None,
18422 judges: 0,
18423 present: 0,
18424 quorum: 0,
18425 met_quorum: true,
18426 uncontested: Some("only candidate A produced a change".to_owned()),
18427 });
18428 state.reviews = vec![ReviewRound {
18429 round: 1,
18430 head: "deadbeef".to_owned(),
18431 verified_head: None,
18432 verified_at: None,
18433 reviews: Vec::new(),
18434 e2e: Vec::new(),
18435 fix: None,
18436 blocking: 0,
18437 answered: 0,
18438 expected: 0,
18439 clean: true,
18440 verify_retried: false,
18441 e2e_deferred: false,
18442 e2e_defer_reason: None,
18443 progressed: false,
18444 vote_split: false,
18445 reconsideration: Vec::new(),
18446 verdict: None,
18447 }];
18448
18449 let mut runner = Runner {
18450 state,
18451 roles: ResolvedRoles {
18452 implementers: Vec::new(),
18453 judges: Vec::new(),
18454 reviewers: Vec::new(),
18455 fixer: None,
18456 conductor: conductor(),
18457 implementer_roster: Vec::new(),
18458 judge_roster: Vec::new(),
18459 reviewer_roster: Vec::new(),
18460 },
18461 sem: Arc::new(Semaphore::new(1)),
18462 pause: Pause::new(),
18463 interrupt: Pause::new(),
18464 };
18465
18466 let started_marker = repo.join("started.marker");
18467 let release_marker = repo.join("release.marker");
18468 let poller = tokio::spawn(async move {
18469 for _ in 0..100 {
18474 if started_marker.exists()
18475 && let Ok(s) = crate::run::RunState::load(&run_id)
18476 && let Some(a) = s.active.get("gate")
18477 {
18478 std::fs::write(&release_marker, b"go").expect("release marker");
18479 return Some(a.clone());
18480 }
18481 tokio::time::sleep(Duration::from_millis(50)).await;
18482 }
18483 None
18484 });
18485
18486 runner.gate().await.expect("gate");
18487 let captured = poller.await.expect("poller task");
18488 let captured = captured.expect(
18489 "the poller never saw a `gate` task entry in run.json while the command was \
18490 still blocked on its own release marker",
18491 );
18492
18493 assert_eq!(captured.task.as_deref(), Some("gate"));
18494 assert_eq!(captured.node, "gate");
18495 assert_eq!(captured.index, Some(1));
18496 assert_eq!(captured.total, Some(1));
18497 assert!(
18498 captured
18499 .command
18500 .as_deref()
18501 .is_some_and(|c| c.contains("started.marker")),
18502 "{captured:?}"
18503 );
18504
18505 assert!(
18506 runner.state.active.is_empty(),
18507 "the entry must be cleared once the command actually finished: {:?}",
18508 runner.state.active
18509 );
18510 assert!(runner.state.gate_ran);
18511 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18512 }
18513
18514 #[tokio::test]
18517 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18518 use crate::verdict::{Finding, ReviewVote, Severity};
18519 crate::run::pin_test_home();
18520 let tmp = tempfile::tempdir().expect("tempdir");
18521 let repo = tmp.path().join("repo");
18522 std::fs::create_dir_all(&repo).unwrap();
18523 init_repo(&repo);
18524
18525 for (severity, vote, expect) in [
18526 (Severity::Major, ReviewVote::Reject, true),
18527 (Severity::Minor, ReviewVote::Reject, false),
18528 (Severity::Major, ReviewVote::Approve, false),
18529 ] {
18530 let mut round = review_round(false, 1, 1, 1, false, true);
18531 round.reviews = vec![ReviewRecord {
18532 reviewer: 1,
18533 agent: "alpha".to_owned(),
18534 summary: String::new(),
18535 findings: vec![Finding {
18536 id: "R1-1-1".to_owned(),
18537 severity,
18538 file: None,
18539 line: None,
18540 title: "t".to_owned(),
18541 detail: String::new(),
18542 }],
18543 vote: Some(vote),
18544 failed: None,
18545 duration_ms: 0,
18546 attempts: 0,
18547 }];
18548 let mut state = RunState::new(
18549 repo.clone(),
18550 "main".to_owned(),
18551 "deadbeef".to_owned(),
18552 "task".to_owned(),
18553 Config::default(),
18554 );
18555 state.reviews = vec![round];
18556 let mut runner = Runner {
18557 state,
18558 roles: ResolvedRoles {
18559 implementers: Vec::new(),
18560 judges: Vec::new(),
18561 reviewers: Vec::new(),
18562 fixer: None,
18563 conductor: conductor(),
18564 implementer_roster: Vec::new(),
18565 judge_roster: Vec::new(),
18566 reviewer_roster: Vec::new(),
18567 },
18568 sem: Arc::new(Semaphore::new(1)),
18569 pause: Pause::new(),
18570 interrupt: Pause::new(),
18571 };
18572 let shell = runner.state.config.shell();
18573 runner
18574 .stop_reviewing("round budget spent", &shell, &repo)
18575 .await
18576 .expect("stop_reviewing");
18577 assert_eq!(runner.state.status, RunStatus::Gating);
18578 assert_eq!(
18579 runner.state.contested_handoff.is_some(),
18580 expect,
18581 "{severity:?} + {vote:?}"
18582 );
18583 }
18584 }
18585
18586 #[tokio::test]
18599 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18600 crate::run::pin_test_home();
18601 let home = crate::run::home();
18602
18603 let tmp = tempfile::tempdir().expect("tempdir");
18604 let repo = tmp.path().join("repo");
18605 std::fs::create_dir_all(&repo).unwrap();
18606 init_repo(&repo);
18607 let head = crate::git::rev_parse(&repo, "HEAD")
18608 .await
18609 .expect("rev-parse");
18610 let cache_dir = tmp.path().join("target");
18613
18614 let mut config = Config::default();
18615 config.verify.e2e = vec![format!(
18616 "CARGO_TARGET_DIR='{}' test -f README.md",
18617 cache_dir.display()
18618 )];
18619 config.graph.review_rounds = 1;
18620 config.graph.timeout_verify = Some(2);
18623 config.graph.timeout_resource_wait = 2;
18624
18625 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18626 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18627 .expect("no io error acquiring directly")
18628 {
18629 crate::cache::AcquireOutcome::Acquired(g) => g,
18630 crate::cache::AcquireOutcome::Busy(b) => {
18631 panic!("expected the direct acquire to win the lease first: {b:?}")
18632 }
18633 };
18634
18635 let mut state = RunState::new(
18636 repo.clone(),
18637 "main".to_owned(),
18638 head.clone(),
18639 "task".to_owned(),
18640 config,
18641 );
18642 state.candidates = vec![Candidate {
18643 index: 0,
18644 label: 'A',
18645 agent: "alpha".to_owned(),
18646 branch: "does-not-exist".to_owned(),
18647 worktree: repo.clone(),
18648 summary: String::new(),
18649 stat: String::new(),
18650 files: 0,
18651 commits: 0,
18652 empty: false,
18653 failed: None,
18654 verified_noop: None,
18655 duration_ms: 0,
18656 folded: false,
18657 }];
18658 state.tally = Some(Tally {
18659 first_choice: BTreeMap::from([('A', 1)]),
18660 borda: BTreeMap::new(),
18661 winner: 'A',
18662 rankings: 1,
18663 unanimous_initial: true,
18664 deliberated: false,
18665 changed_votes: 0,
18666 unanimous_final: true,
18667 tie_break: None,
18668 judges: 0,
18669 present: 0,
18670 quorum: 0,
18671 met_quorum: true,
18672 uncontested: Some("only candidate A produced a change".to_owned()),
18673 });
18674 state.reviews = vec![ReviewRound {
18678 round: 1,
18679 head: head.clone(),
18680 verified_head: None,
18681 verified_at: None,
18682 reviews: Vec::new(),
18683 e2e: Vec::new(),
18684 fix: None,
18685 blocking: 1,
18686 answered: 1,
18687 expected: 1,
18688 clean: false,
18689 verify_retried: false,
18690 e2e_deferred: true,
18691 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18692 progressed: false,
18693 vote_split: false,
18694 reconsideration: Vec::new(),
18695 verdict: None,
18696 }];
18697
18698 let mut runner = Runner {
18699 state,
18700 roles: ResolvedRoles {
18701 implementers: Vec::new(),
18702 judges: Vec::new(),
18703 reviewers: Vec::new(),
18704 fixer: None,
18705 conductor: conductor(),
18706 implementer_roster: Vec::new(),
18707 judge_roster: Vec::new(),
18708 reviewer_roster: Vec::new(),
18709 },
18710 sem: Arc::new(Semaphore::new(1)),
18711 pause: Pause::new(),
18712 interrupt: Pause::new(),
18713 };
18714
18715 let shell = runner.state.config.shell();
18716 runner
18717 .stop_reviewing("round budget spent", &shell, &repo)
18718 .await
18719 .expect("stop_reviewing");
18720
18721 let last = runner.state.reviews.last().expect("round record");
18722 assert_eq!(
18723 last.e2e_status(),
18724 E2eStatus::ResourceBlocked,
18725 "the shared cache is still held; the attempt must read as blocked, not deferred or \
18726 failed: {last:?}"
18727 );
18728 assert_eq!(
18729 last.verified_head.as_deref(),
18730 Some(head.as_str()),
18731 "which commit this attempt targeted is known even though nothing finished checking \
18732 it"
18733 );
18734 let first_attempt_at = last
18735 .verified_at
18736 .expect("when this attempt ran is known too");
18737 assert_ne!(
18738 runner.state.status,
18739 RunStatus::Blocked,
18740 "contention is evidence about the machine, not the patch — it must not settle the \
18741 run as blocked: {:?}",
18742 runner.state.status
18743 );
18744 assert!(
18745 !runner
18746 .state
18747 .events
18748 .iter()
18749 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18750 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18751 runner.state.events
18752 );
18753
18754 runner
18759 .stop_reviewing("round budget spent", &shell, &repo)
18760 .await
18761 .expect("stop_reviewing retry");
18762 assert_eq!(
18763 runner.state.reviews.len(),
18764 1,
18765 "no new round was started: {:?}",
18766 runner.state.reviews
18767 );
18768 let last = runner.state.reviews.last().expect("round record");
18769 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18770 assert!(
18771 last.verified_at.expect("still known") > first_attempt_at,
18772 "a second reentry must be a fresh attempt, not a stale copy of the first"
18773 );
18774 assert_ne!(runner.state.status, RunStatus::Blocked);
18775
18776 held.release();
18777 }
18778
18779 #[cfg(unix)]
18780 #[tokio::test]
18781 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
18782 ask_test_home();
18783 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
18784 let tmp = tempfile::tempdir().unwrap();
18785 let repo = tmp.path().join("repo");
18786 std::fs::create_dir(&repo).unwrap();
18787 init_repo(&repo);
18788 let run_git = |args: &[&str]| {
18789 let out = std::process::Command::new("git")
18790 .args(args)
18791 .current_dir(&repo)
18792 .output()
18793 .unwrap();
18794 assert!(
18795 out.status.success(),
18796 "{args:?}: {}",
18797 String::from_utf8_lossy(&out.stderr)
18798 );
18799 };
18800 let origin = tmp.path().join("origin.git");
18801 run_git(&["init", "--bare", origin.to_str().unwrap()]);
18802 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
18803 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
18804 run_git(&["add", "spec.rs"]);
18805 run_git(&["commit", "-m", "initial spec"]);
18806 run_git(&["push", "origin", "main"]);
18807 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
18808 run_git(&["checkout", "-b", "fix/review"]);
18809 std::fs::write(
18810 repo.join("check.rs"),
18811 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
18812 )
18813 .unwrap();
18814 run_git(&["add", "check.rs"]);
18815 run_git(&["commit", "-m", "use spec"]);
18816 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
18817 run_git(&["checkout", "main"]);
18818 std::fs::write(
18819 repo.join("spec.rs"),
18820 "struct AgentSpec { max_turn_secs: u64 }\n",
18821 )
18822 .unwrap();
18823 run_git(&["add", "spec.rs"]);
18824 run_git(&["commit", "-m", "extend spec"]);
18825 run_git(&["push", "origin", "main"]);
18826 run_git(&["checkout", "fix/review"]);
18827
18828 let mut runner = runner_at(RunStatus::Reviewing);
18829 runner.state.repo = repo.clone();
18830 runner.state.base_commit = base;
18831 runner.state.pr_head = Some("fix/review".into());
18832 runner.state.config.graph.review_rounds = max_rounds;
18833 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
18834 runner.state.config.graph.sessions = false;
18835 runner.state.config.disk.min_free_bytes = 0;
18836 runner.state.config.verify.e2e =
18837 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
18838 runner.state.candidates = vec![Candidate {
18839 index: 0,
18840 label: 'A',
18841 agent: "fixture".into(),
18842 branch: "fix/review".into(),
18843 worktree: repo.clone(),
18844 summary: String::new(),
18845 stat: String::new(),
18846 files: 1,
18847 commits: 1,
18848 empty: false,
18849 failed: None,
18850 verified_noop: None,
18851 duration_ms: 0,
18852 folded: false,
18853 }];
18854 runner.state.tally = Some(Tally {
18855 first_choice: BTreeMap::new(),
18856 borda: BTreeMap::new(),
18857 winner: 'A',
18858 rankings: 0,
18859 unanimous_initial: true,
18860 deliberated: false,
18861 changed_votes: 0,
18862 unanimous_final: true,
18863 tie_break: None,
18864 judges: 0,
18865 present: 0,
18866 quorum: 0,
18867 met_quorum: true,
18868 uncontested: Some("review-only".into()),
18869 });
18870 let mut last = review_round(false, 0, 1, 1, false, false);
18871 last.head = reviewed.clone();
18872 last.e2e_deferred = true;
18873 last.e2e[0].resource_blocked = true;
18874 last.e2e[0].output_tail = "waiting for the shared build cache".into();
18875 if max_rounds == 1 {
18876 last.e2e.clear();
18877 }
18878 runner.state.reviews = vec![last];
18879 let calls = tmp.path().join("fix-calls");
18880 let mut fixer = spec("repair");
18881 fixer
18882 .env
18883 .insert("FIX_CALLS".into(), calls.display().to_string());
18884 fixer.command = vec!["sh".into(), "-c".into(), r#"
18885 echo fix >> "$FIX_CALLS"
18886 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18887 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18888 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18889 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18890 "#.into()];
18891 let mut reviewer = spec("reviewer");
18892 reviewer.command = vec![
18893 "sh".into(),
18894 "-c".into(),
18895 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18896 .into(),
18897 ];
18898 runner.roles.fixer = Some(vec![fixer]);
18899 runner.roles.reviewers = vec![reviewer];
18900
18901 runner.sync_to_base().await.unwrap();
18902 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18903 assert_ne!(rebased, reviewed);
18904 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18905 if reentry {
18906 let shell = runner.state.config.shell();
18909 runner
18910 .stop_unresolved_review("deferred verification", &shell, &repo)
18911 .await
18912 .unwrap();
18913 assert!(runner.catchup_repairable());
18914 assert!(!calls.exists());
18915 runner.state = RunState::load(&runner.state.id).unwrap();
18916 assert!(runner.catchup_repairable());
18917 }
18918 runner.review_loop().await.unwrap();
18919 let first = &runner.state.reviews[0];
18920 assert_eq!(first.head, reviewed);
18921 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18922 assert!(
18923 first.verify_retried,
18924 "a reproduced build failure gets exactly one retry: {:?}",
18925 first.e2e
18926 );
18927 assert!(first.e2e[0].output_tail.contains("E0063"));
18928 assert!(!runner.catchup_repairable());
18929 if max_rounds == 2 {
18930 assert_eq!(
18931 runner.state.status,
18932 RunStatus::Gating,
18933 "{}",
18934 crate::report::run(&runner.state)
18935 );
18936 assert_eq!(runner.state.reviews.len(), 2);
18937 assert!(first.fix.as_ref().unwrap().committed);
18938 assert!(runner.state.reviews[1].clean);
18939 assert_eq!(
18940 runner
18941 .state
18942 .reviews
18943 .iter()
18944 .filter(|r| r.fix.is_some())
18945 .count(),
18946 1
18947 );
18948 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18949 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18950 } else {
18951 assert_eq!(runner.state.status, RunStatus::Blocked);
18952 assert!(first.fix.is_none());
18953 assert!(!calls.exists());
18954 let recovery = crate::recovery::assess(
18955 "prior",
18956 Some(&runner.state),
18957 &runner.landing_base(),
18958 crate::recovery::Relation::Diverged,
18959 );
18960 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18961 let reasons = recovery.reasons.join("\n");
18962 assert!(
18963 reasons.contains(&reviewed) && reasons.contains(&rebased),
18964 "{reasons}"
18965 );
18966 assert!(
18967 reasons.contains("base sync to")
18968 && reasons.contains("additional review is required"),
18969 "{reasons}"
18970 );
18971 }
18972 }
18973 }
18974
18975 #[tokio::test]
18987 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18988 crate::run::pin_test_home();
18989 let home = crate::run::home();
18990
18991 let tmp = tempfile::tempdir().expect("tempdir");
18992 let repo = tmp.path().join("repo");
18993 std::fs::create_dir_all(&repo).unwrap();
18994 init_repo(&repo);
18995 let head = crate::git::rev_parse(&repo, "HEAD")
18996 .await
18997 .expect("rev-parse");
18998 let cache_dir = tmp.path().join("target");
18999
19000 let mut config = Config::default();
19001 config.verify.e2e = vec![format!(
19002 "CARGO_TARGET_DIR='{}' test -f README.md",
19003 cache_dir.display()
19004 )];
19005 config.graph.review_rounds = 1;
19006 config.graph.timeout_verify = Some(2);
19007 config.graph.timeout_resource_wait = 2;
19008
19009 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19010 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19011 .expect("no io error acquiring directly")
19012 {
19013 crate::cache::AcquireOutcome::Acquired(g) => g,
19014 crate::cache::AcquireOutcome::Busy(b) => {
19015 panic!("expected the direct acquire to win the lease first: {b:?}")
19016 }
19017 };
19018
19019 let mut state = RunState::new(
19020 repo.clone(),
19021 "main".to_owned(),
19022 head.clone(),
19023 "task".to_owned(),
19024 config,
19025 );
19026 state.candidates = vec![Candidate {
19027 index: 0,
19028 label: 'A',
19029 agent: "alpha".to_owned(),
19030 branch: "does-not-exist".to_owned(),
19031 worktree: repo.clone(),
19032 summary: String::new(),
19033 stat: String::new(),
19034 files: 0,
19035 commits: 0,
19036 empty: false,
19037 failed: None,
19038 verified_noop: None,
19039 duration_ms: 0,
19040 folded: false,
19041 }];
19042 state.tally = Some(Tally {
19043 first_choice: BTreeMap::from([('A', 1)]),
19044 borda: BTreeMap::new(),
19045 winner: 'A',
19046 rankings: 1,
19047 unanimous_initial: true,
19048 deliberated: false,
19049 changed_votes: 0,
19050 unanimous_final: true,
19051 tie_break: None,
19052 judges: 0,
19053 present: 0,
19054 quorum: 0,
19055 met_quorum: true,
19056 uncontested: Some("only candidate A produced a change".to_owned()),
19057 });
19058 state.reviews = vec![ReviewRound {
19062 round: 1,
19063 head: head.clone(),
19064 verified_head: Some(head.clone()),
19065 verified_at: Some(jiff::Timestamp::now()),
19066 reviews: Vec::new(),
19067 e2e: vec![CommandOutcome {
19068 command: format!(
19069 "CARGO_TARGET_DIR='{}' test -f README.md",
19070 cache_dir.display()
19071 ),
19072 code: None,
19073 output_tail: "waiting for the shared build cache".to_owned(),
19074 duration_ms: 0,
19075 resource_blocked: true,
19076 }],
19077 fix: None,
19078 blocking: 1,
19079 answered: 1,
19080 expected: 1,
19081 clean: false,
19082 verify_retried: false,
19083 e2e_deferred: false,
19084 e2e_defer_reason: None,
19085 progressed: false,
19086 vote_split: false,
19087 reconsideration: Vec::new(),
19088 verdict: None,
19089 }];
19090
19091 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
19092 let mut runner = Runner {
19093 state,
19094 roles: ResolvedRoles {
19095 implementers: Vec::new(),
19096 judges: Vec::new(),
19097 reviewers: Vec::new(),
19098 fixer: None,
19099 conductor: conductor(),
19100 implementer_roster: Vec::new(),
19101 judge_roster: Vec::new(),
19102 reviewer_roster: Vec::new(),
19103 },
19104 sem: Arc::new(Semaphore::new(1)),
19105 pause: Pause::new(),
19106 interrupt: Pause::new(),
19107 };
19108
19109 runner.review_loop().await.expect("review_loop");
19114
19115 assert_eq!(
19116 runner.state.reviews.len(),
19117 1,
19118 "no new round was started on top of the unresolved one: {:?}",
19119 runner.state.reviews
19120 );
19121 let last = &runner.state.reviews[0];
19122 assert_eq!(
19123 last.e2e_status(),
19124 E2eStatus::ResourceBlocked,
19125 "still contended: {last:?}"
19126 );
19127 assert!(
19128 last.verified_at.expect("still known") > first_attempt_at,
19129 "review_loop must have actually retried the check, not left it exactly as found"
19130 );
19131 assert_ne!(
19132 runner.state.status,
19133 RunStatus::Blocked,
19134 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
19135 runner.state.status
19136 );
19137
19138 held.release();
19139 }
19140
19141 #[tokio::test]
19142 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
19143 crate::run::pin_test_home();
19144 for status in [RunStatus::Blocked, RunStatus::Gating] {
19145 let mut runner = runner_at(status);
19146 runner.state.config.graph.land = true;
19147 runner.state.gate_ran = true;
19148 runner.state.merge = Some(MergeOutcome {
19149 mode: MergeMode::Pr,
19150 ok: false,
19151 detail: "the fixer produced no commit".into(),
19152 empty: false,
19153 });
19154 let url = "https://example.invalid/o/r/pull/1";
19155 runner.state.pr = Some(crate::run::PrRecord {
19156 url: url.into(),
19157 number: 1,
19158 state: "open".into(),
19159 checks: "red".into(),
19160 round: 1,
19161 rounds: 1,
19162 red_at_merge: Vec::new(),
19163 });
19164 runner.state.save().unwrap();
19165 runner.execute_graph().await.unwrap();
19169 assert_eq!(runner.state.status, RunStatus::Blocked);
19170 assert!(runner.state.land_sync_pending);
19171 assert!(runner.state.events.iter().any(|event|
19172 event.node == "land" && event.message == format!("watching {url}")));
19173 assert_eq!(
19174 runner.state.merge.as_ref().unwrap().detail,
19175 "the fixer produced no commit"
19176 );
19177 }
19178 }
19179
19180 #[tokio::test]
19181 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
19182 crate::run::pin_test_home();
19183 let tmp = tempfile::tempdir().expect("tempdir");
19184 let repo = tmp.path().join("repo");
19185 std::fs::create_dir_all(&repo).unwrap();
19186 init_repo(&repo);
19187
19188 let mut config = Config::default();
19189 config.merge.mode = MergeMode::Pr;
19190 config.graph.land = true;
19191 config.graph.land_approval = false;
19192
19193 let mut state = RunState::new(
19194 repo.clone(),
19195 "main".to_owned(),
19196 "deadbeef".to_owned(),
19197 "task".to_owned(),
19198 config,
19199 );
19200 state.candidates = vec![Candidate {
19201 index: 0,
19202 label: 'A',
19203 agent: "alpha".to_owned(),
19204 branch: "does-not-exist".to_owned(),
19205 worktree: repo.clone(),
19206 summary: String::new(),
19207 stat: String::new(),
19208 files: 0,
19209 commits: 0,
19210 empty: false,
19211 failed: None,
19212 verified_noop: None,
19213 duration_ms: 0,
19214 folded: false,
19215 }];
19216 state.tally = Some(Tally {
19217 first_choice: BTreeMap::from([('A', 1)]),
19218 borda: BTreeMap::new(),
19219 winner: 'A',
19220 rankings: 1,
19221 unanimous_initial: true,
19222 deliberated: false,
19223 changed_votes: 0,
19224 unanimous_final: true,
19225 tie_break: None,
19226 judges: 0,
19227 present: 0,
19228 quorum: 0,
19229 met_quorum: true,
19230 uncontested: Some("only candidate A produced a change".to_owned()),
19231 });
19232 state.reviews = vec![ReviewRound {
19233 round: 1,
19234 head: "deadbeef".to_owned(),
19235 verified_head: None,
19236 verified_at: None,
19237 reviews: Vec::new(),
19238 e2e: Vec::new(),
19239 fix: None,
19240 blocking: 0,
19241 answered: 0,
19242 expected: 0,
19243 clean: true,
19244 verify_retried: false,
19245 e2e_deferred: false,
19246 e2e_defer_reason: None,
19247 progressed: false,
19248 vote_split: false,
19249 reconsideration: Vec::new(),
19250 verdict: None,
19251 }];
19252 state.gate = vec![CommandOutcome {
19253 command: "test".to_owned(),
19254 code: Some(0),
19255 output_tail: String::new(),
19256 duration_ms: 0,
19257 resource_blocked: false,
19258 }];
19259 state.gate_ran = true;
19260 state.status = RunStatus::Landing;
19264 state.merge = Some(MergeOutcome {
19265 mode: MergeMode::Pr,
19266 ok: true,
19267 detail: "https://example.invalid/x/y/pull/1".to_owned(),
19268 empty: false,
19269 });
19270
19271 ask_test_home();
19275 state.save().expect("save landing run");
19278 let store = ask::Questions::open();
19279 let q = ask_open_question(&store, &state.id);
19280
19281 let mut runner = Runner {
19282 state,
19283 roles: ResolvedRoles {
19284 implementers: Vec::new(),
19285 judges: Vec::new(),
19286 reviewers: Vec::new(),
19287 fixer: None,
19288 conductor: conductor(),
19289 implementer_roster: Vec::new(),
19290 judge_roster: Vec::new(),
19291 reviewer_roster: Vec::new(),
19292 },
19293 sem: Arc::new(Semaphore::new(1)),
19294 pause: Pause::new(),
19295 interrupt: Pause::new(),
19296 };
19297
19298 runner.execute().await.expect("execute");
19303
19304 assert_eq!(
19305 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19306 Some("https://example.invalid/x/y/pull/1"),
19307 "reentry must not push again or open a second pull request over the \
19308 one `land` is already watching"
19309 );
19310 assert_ne!(
19311 runner.state.status,
19312 RunStatus::Landing,
19313 "land could not actually reach the fake pull request, so it must \
19314 have given up rather than left the run silently parked forever"
19315 );
19316 assert_eq!(runner.state.status, RunStatus::Blocked);
19320 assert!(
19321 store.get(&q.id).unwrap().status.open(),
19322 "Blocked is still alive; settle_questions must have been a no-op here"
19323 );
19324 }
19325
19326 fn state_with_round(round: ReviewRound) -> RunState {
19327 let mut s = RunState::new(
19328 PathBuf::from("/repo"),
19329 "main".to_owned(),
19330 "abc1234".to_owned(),
19331 "add retries".to_owned(),
19332 Config::default(),
19333 );
19334 s.reviews = vec![round];
19335 s
19336 }
19337
19338 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19339 crate::verdict::Finding {
19340 id: id.to_owned(),
19341 severity,
19342 file: None,
19343 line: None,
19344 title: title.to_owned(),
19345 detail: String::new(),
19346 }
19347 }
19348
19349 #[test]
19350 fn pr_body_names_open_findings_and_declined_ones() {
19351 let round = ReviewRound {
19352 round: 2,
19353 head: "deadbee".to_owned(),
19354 verified_head: None,
19355 verified_at: None,
19356 reviews: vec![ReviewRecord {
19357 attempts: 0,
19358 reviewer: 1,
19359 agent: "alpha".to_owned(),
19360 summary: String::new(),
19361 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19362 vote: None,
19363 failed: None,
19364 duration_ms: 0,
19365 }],
19366 e2e: vec![CommandOutcome {
19367 command: "cargo test".to_owned(),
19368 code: Some(0),
19369 output_tail: String::new(),
19370 duration_ms: 0,
19371 resource_blocked: false,
19372 }],
19373 verify_retried: false,
19374 e2e_deferred: false,
19375 e2e_defer_reason: None,
19376 fix: Some(FixRecord {
19377 agent: "alpha".to_owned(),
19378 addressed: Vec::new(),
19379 rejected: vec![crate::verdict::Rejection {
19380 id: "R1-1-1".to_owned(),
19381 why: "not reachable from any caller".to_owned(),
19382 }],
19383 notes: String::new(),
19384 committed: true,
19385 failed: None,
19386 duration_ms: 0,
19387 continuation: None,
19388 }),
19389 blocking: 0,
19390 answered: 1,
19391 expected: 1,
19392 clean: false,
19393 progressed: true,
19394 vote_split: false,
19395 reconsideration: Vec::new(),
19396 verdict: None,
19397 };
19398 let state = state_with_round(round);
19399 let body = pr_message(&state, 'A').body;
19400
19401 assert!(body.contains("add retries"), "the task must still be there");
19402 assert!(body.contains("R2-1-1"), "{body}");
19403 assert!(!body.contains("unused import"), "{body}");
19404 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19405 assert!(
19406 !body.contains("not reachable from any caller"),
19407 "the reason it was declined: {body}"
19408 );
19409 }
19410
19411 #[test]
19412 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19413 let round = ReviewRound {
19414 round: 1,
19415 head: "deadbee".to_owned(),
19416 verified_head: None,
19417 verified_at: None,
19418 reviews: vec![ReviewRecord {
19419 attempts: 0,
19420 reviewer: 1,
19421 agent: "alpha".to_owned(),
19422 summary: String::new(),
19423 findings: Vec::new(),
19424 vote: None,
19425 failed: None,
19426 duration_ms: 0,
19427 }],
19428 e2e: Vec::new(),
19429 verify_retried: false,
19430 e2e_deferred: false,
19431 e2e_defer_reason: None,
19432 fix: None,
19433 blocking: 0,
19434 answered: 1,
19435 expected: 1,
19436 clean: true,
19437 progressed: false,
19438 vote_split: false,
19439 reconsideration: Vec::new(),
19440 verdict: None,
19441 };
19442 let state = state_with_round(round);
19443 let body = pr_message(&state, 'A').body;
19444 assert!(!body.contains("Open review findings"), "{body}");
19445 assert!(!body.contains("Declined"), "{body}");
19446 }
19447
19448 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19449 let mut state = RunState::new(
19450 PathBuf::from("/repo"),
19451 "main".to_owned(),
19452 "abc1234".to_owned(),
19453 instruction.to_owned(),
19454 Config::default(),
19455 );
19456 state.candidates.push(Candidate {
19457 index: 0,
19458 label: 'A',
19459 agent: "alpha".to_owned(),
19460 branch: "magi/x/A".to_owned(),
19461 worktree: PathBuf::from("/wt"),
19462 summary: summary.to_owned(),
19463 stat: String::new(),
19464 files: 1,
19465 commits: 1,
19466 empty: false,
19467 failed: None,
19468 verified_noop: None,
19469 folded: false,
19470 duration_ms: 0,
19471 });
19472 state
19473 }
19474
19475 fn review_state(subjects: &[&str]) -> RunState {
19476 let mut state = state_with_summary(
19477 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19478 "",
19479 );
19480 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19481 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19482 state
19483 }
19484
19485 #[tokio::test]
19488 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19489 ask_test_home();
19490 let tmp = tempfile::tempdir().unwrap();
19491 let repo = tmp.path().join("repo");
19492 std::fs::create_dir_all(&repo).unwrap();
19493 init_repo(&repo);
19494 let origin = tmp.path().join("origin.git");
19495 let g = |dir: &Path, args: &[&str]| {
19496 let out = std::process::Command::new("git")
19497 .args(args)
19498 .current_dir(dir)
19499 .quiet()
19500 .output()
19501 .expect("spawn git");
19502 assert!(
19503 out.status.success(),
19504 "git {args:?}: {}",
19505 String::from_utf8_lossy(&out.stderr)
19506 );
19507 };
19508 g(
19509 tmp.path(),
19510 &[
19511 "clone",
19512 "--bare",
19513 "-q",
19514 repo.to_str().unwrap(),
19515 origin.to_str().unwrap(),
19516 ],
19517 );
19518 g(
19519 &repo,
19520 &["remote", "add", "origin", origin.to_str().unwrap()],
19521 );
19522 let c1 = git::rev_parse(&repo, "main").await.unwrap();
19523
19524 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19526 g(&repo, &["add", "-A"]);
19527 g(
19528 &repo,
19529 &["commit", "-q", "-m", "chore(deps): update a crate"],
19530 );
19531 g(&repo, &["push", "-q", "origin", "main"]);
19532 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19533 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19534 g(&repo, &["add", "-A"]);
19535 g(
19536 &repo,
19537 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19538 );
19539 g(&repo, &["checkout", "-q", "main"]);
19540
19541 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19542 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19543 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19545 assert_eq!(fallback, c1);
19546
19547 let mut state = review_state(&[
19548 "chore(deps): update a crate",
19549 "fix(daemon): apply a chosen action",
19550 ]);
19551 state.repo = repo.clone();
19552 state.base_branch = "main".to_owned();
19553 state.base_commit = c1;
19554 refresh_reviewed_commits(&mut state, "feat/own").await;
19555 assert_eq!(
19556 state.reviewed_commits,
19557 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19558 );
19559 assert_eq!(
19560 review_title(&state).as_deref(),
19561 Some("fix(daemon): apply a chosen action")
19562 );
19563 assert_eq!(
19564 leaked_subjects(&state, "feat/own").await,
19565 Some(vec!["chore(deps): update a crate".to_owned()])
19566 );
19567 state.config.merge.remote = "nowhere".to_owned();
19570 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19571 }
19572
19573 struct SummaryFixture {
19577 _tmp: tempfile::TempDir,
19578 repo: PathBuf,
19579 reply: PathBuf,
19580 runner: Runner,
19581 }
19582
19583 fn summary_fixture() -> SummaryFixture {
19584 ask_test_home();
19585 let tmp = tempfile::tempdir().unwrap();
19586 let repo = tmp.path().join("repo");
19587 std::fs::create_dir_all(&repo).unwrap();
19588 init_repo(&repo);
19589 let origin = tmp.path().join("origin.git");
19590 let g = |args: &[&str]| {
19591 let out = std::process::Command::new("git")
19592 .args(args)
19593 .current_dir(&repo)
19594 .quiet()
19595 .output()
19596 .expect("spawn git");
19597 assert!(
19598 out.status.success(),
19599 "git {args:?}: {}",
19600 String::from_utf8_lossy(&out.stderr)
19601 );
19602 };
19603 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19604 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19605 g(&["push", "-q", "origin", "main"]);
19606 g(&["checkout", "-q", "-b", "feat/own"]);
19607 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19608 g(&["add", "-A"]);
19609 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19610 let reply = tmp.path().join("reply.txt");
19611 std::fs::write(
19612 &reply,
19613 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19614 )
19615 .unwrap();
19616 let mut runner = runner_at(RunStatus::Gating);
19617 let mut state = review_state(&["feat(own): add the own file"]);
19618 state.repo = repo.clone();
19619 state.base_branch = "main".to_owned();
19620 state.candidates[0].branch = "feat/own".to_owned();
19621 state.candidates[0].worktree = repo.clone();
19622 let mut author = spec("alpha");
19623 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19624 author
19625 .env
19626 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19627 state.config.agents = vec![author];
19628 runner.state = state;
19629 SummaryFixture {
19630 _tmp: tmp,
19631 repo,
19632 reply,
19633 runner,
19634 }
19635 }
19636
19637 async fn posted(f: &mut SummaryFixture) -> PrMessage {
19638 let winner = f.runner.state.candidates[0].clone();
19639 f.runner
19640 .guarded_pr_message(&winner, None, true)
19641 .await
19642 .unwrap()
19643 .expect("an English description never parks")
19644 }
19645
19646 fn clean_round_with_minor_findings() -> ReviewRound {
19647 let mut round = passing_round(2, None);
19648 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19649 round.reviews = vec![ReviewRecord {
19650 attempts: 0,
19651 reviewer: 1,
19652 agent: "alpha".into(),
19653 summary: "内部レビュー".into(),
19654 findings: vec![
19655 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19656 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19657 ],
19658 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19659 failed: None,
19660 duration_ms: 0,
19661 }];
19662 round
19663 }
19664
19665 #[tokio::test]
19666 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19667 for review_only in [false, true] {
19668 let mut f = summary_fixture();
19669 if !review_only {
19670 f.runner.state.candidates[0].agent = "alpha".into();
19671 }
19672 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19673 assert!(f.runner.state.reviews[0].clean);
19674 assert!(f.runner.state.reviews[0].fix.is_none());
19675 assert!(f.runner.state.open_findings().is_empty());
19676 assert_eq!(
19677 crate::pr_outcome::required_finding_ids(&f.runner.state),
19678 ["R2-1-1", "R2-1-2"]
19679 );
19680 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19681 let evidence = crate::pr_outcome::summary_evidence(
19682 &f.runner.state,
19683 crate::pr_outcome::FinalHead::At(&head),
19684 );
19685 assert!(
19686 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19687 );
19688 let omitted = posted(&mut f).await;
19690 assert!(omitted.body.contains("No description"));
19691 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19692 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19693 assert!(f.runner.state.change_summary.is_none());
19694 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19695 let described = posted(&mut f).await;
19696 assert!(described.body.contains("improve documentation for callers"));
19697 assert!(described.body.contains("remaining edge case"));
19698 assert!(described.body.contains("## Open review findings"));
19699 assert!(!described.body.contains("未修正"));
19700 }
19701 }
19702
19703 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19704 let mut f = summary_fixture();
19705 if !review_only {
19706 f.runner.state.candidates[0].agent = "alpha".into();
19707 }
19708 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19709 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19710 fixed
19711 .fix
19712 .as_mut()
19713 .unwrap()
19714 .rejected
19715 .push(crate::verdict::Rejection {
19716 id: "R1-1-1".into(),
19717 why: "呼び出されないコードのため辞退".into(),
19718 });
19719 let mut clean = clean_round_with_minor_findings();
19720 clean.verified_head = Some(head.clone());
19721 f.runner.state.reviews = vec![fixed, clean];
19722 ok_gate(&mut f.runner.state, Some(&head));
19723 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19724 f.runner.state.config.agents.push(scripted("beta", reply));
19725 f.runner.state.config.roles.language_judge =
19726 Some(crate::config::AgentChoice::One("beta".into()));
19727 f
19728 }
19729
19730 #[tokio::test]
19731 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19732 for review_only in [false, true] {
19733 let mut f = rewrite_fixture_with_findings(review_only,
19734 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19735 let m = posted(&mut f).await;
19736 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19737 let expected =
19738 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19739 assert!(m.body.contains(&expected));
19740 assert!(m.body.contains("## Review rounds"));
19741 assert!(
19742 m.body
19743 .contains("Gate (`verify.gate`): passed on the final commit")
19744 );
19745 assert!(
19746 m.body
19747 .contains("Tests (`verify.e2e`): passed on the final commit")
19748 );
19749 let c = f.runner.state.change_summary.as_ref().unwrap();
19750 assert!(
19751 m.body
19752 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19753 );
19754 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19755 assert!(crate::pr_outcome::covers_required_findings(
19756 &f.runner.state,
19757 &m.body
19758 ));
19759 assert!(m.body.contains("documentation remains incomplete"));
19760 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19761 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19762 assert!(f.runner.state.github_text.is_none());
19763 }
19764 }
19765
19766 #[tokio::test]
19767 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19768 for review_only in [false, true] {
19769 for reply in [
19770 "TITLE: fix: public description\nA short English description.",
19771 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19772 ] {
19773 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19774 let winner = f.runner.state.candidates[0].clone();
19775 let posted = f
19776 .runner
19777 .guarded_pr_message(&winner, None, true)
19778 .await
19779 .unwrap();
19780 assert!(posted.is_none());
19781 assert!(f.runner.state.parked);
19782 let withheld = f.runner.state.github_text.as_ref().unwrap();
19783 assert!(
19784 withheld.rewrite.is_none(),
19785 "an incomplete rewrite must not be selectable"
19786 );
19787 let q = ask::Questions::open()
19788 .get(withheld.question.as_ref().unwrap())
19789 .unwrap();
19790 assert!(
19791 !q.choices
19792 .contains(&crate::github_text::USE_REWRITE.to_owned())
19793 );
19794 assert!(
19795 q.detail.contains("R1-1-1")
19796 && q.detail.contains("R2-1-1")
19797 && q.detail.contains("R2-1-2")
19798 );
19799 }
19800 }
19801 }
19802
19803 #[tokio::test]
19804 async fn japanese_reports_stay_local_in_both_public_run_modes() {
19805 for review_only in [false, true] {
19806 let mut f = summary_fixture();
19807 if !review_only {
19808 f.runner.state.candidates[0].agent = "alpha".into();
19809 }
19810 f.runner.state.instruction = "内部タスクの説明".into();
19811 f.runner.state.candidates[0].summary = "内部の候補報告".into();
19812 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19813 f.runner.state.reviews = vec![
19814 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
19815 round_with_notes(2, Some("二回目の内部修正報告です。")),
19816 passing_round(3, Some(&head)),
19817 ];
19818 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
19819 ok_gate(&mut f.runner.state, Some(&head));
19820 let captured = f.repo.join("public-prompt.txt");
19821 let author = &mut f.runner.state.config.agents[0];
19822 author.command = vec![
19823 "sh".into(),
19824 "-c".into(),
19825 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19826 ];
19827 author.env.insert(
19828 "PROMPT_FILE".into(),
19829 captured.to_string_lossy().into_owned(),
19830 );
19831 let m = posted(&mut f).await;
19832 assert!(m.body.contains("Adds `own.txt`"));
19833 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
19834 assert!(m.body.contains("R1-1-1"));
19835 assert!(
19836 m.body
19837 .contains("Gate (`verify.gate`): passed on the final commit")
19838 );
19839 assert!(
19840 m.body
19841 .contains("Tests (`verify.e2e`): passed on the final commit")
19842 );
19843 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19844 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
19845 let prompt = std::fs::read_to_string(captured).unwrap();
19846 assert!(prompt.contains(&format!(
19847 "git diff {} {head}",
19848 f.runner.state.change_summary.as_ref().unwrap().base
19849 )));
19850 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
19851 assert_eq!(
19852 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
19853 "二回目の内部修正報告です。"
19854 );
19855 }
19856 }
19857
19858 #[tokio::test]
19859 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
19860 for review_only in [false, true] {
19861 let mut f = summary_fixture();
19862 if !review_only {
19863 f.runner.state.candidates[0].agent = "alpha".into();
19864 }
19865 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
19866 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
19867 std::fs::write(&f.reply, "no usable description").unwrap();
19868 let failed = posted(&mut f).await;
19869 assert!(
19870 failed
19871 .body
19872 .contains("No description of what this change does could be generated")
19873 );
19874 assert!(failed.body.is_ascii());
19875 assert!(f.runner.state.github_text.is_none());
19876 assert!(f.runner.state.change_summary.is_none());
19877 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19878 let evidence = crate::pr_outcome::summary_evidence(
19879 &f.runner.state,
19880 crate::pr_outcome::FinalHead::At(&head),
19881 );
19882 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
19883 assert!(
19884 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19885 .exists()
19886 );
19887 f.runner.state =
19888 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19889 std::fs::write(
19890 &f.reply,
19891 "TITLE: feat(own): add the own file\nAdds the committed own file.",
19892 )
19893 .unwrap();
19894 let retried = posted(&mut f).await;
19895 assert!(retried.body.contains("Adds the committed own file."));
19896 assert_eq!(
19897 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19898 "日本語の修正報告"
19899 );
19900 }
19901 }
19902
19903 #[tokio::test]
19904 async fn same_head_verification_and_findings_invalidate_public_prose() {
19905 let mut f = summary_fixture();
19906 posted(&mut f).await;
19907 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19908 let original = f
19909 .runner
19910 .state
19911 .change_summary
19912 .as_ref()
19913 .unwrap()
19914 .evidence
19915 .clone();
19916 ok_gate(&mut f.runner.state, Some(&head));
19917 std::fs::write(
19918 &f.reply,
19919 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19920 )
19921 .unwrap();
19922 let verified = posted(&mut f).await;
19923 assert!(verified.body.contains("now verified"));
19924 assert_ne!(
19925 f.runner.state.change_summary.as_ref().unwrap().evidence,
19926 original
19927 );
19928 let mut round = round_with_notes(1, Some("内部報告"));
19929 round.reviews.push(ReviewRecord {
19930 attempts: 0,
19931 reviewer: 1,
19932 agent: "alpha".into(),
19933 summary: "内部レビュー".into(),
19934 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19935 vote: None,
19936 failed: None,
19937 duration_ms: 0,
19938 });
19939 round
19940 .fix
19941 .as_mut()
19942 .unwrap()
19943 .rejected
19944 .push(crate::verdict::Rejection {
19945 id: "R1-1-1".into(),
19946 why: "呼び出し元から到達できない".into(),
19947 });
19948 f.runner.state.reviews = vec![round];
19949 let omitted = posted(&mut f).await;
19951 assert!(omitted.body.contains("No description"));
19952 assert!(omitted.body.contains("R1-1-1"));
19953 assert!(!omitted.body.contains("到達"));
19954 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19955 let described = posted(&mut f).await;
19956 assert!(described.body.contains("unreachable from callers"));
19957 assert!(described.body.contains("R1-1-1"));
19958 assert!(!described.body.contains("到達"));
19959 let evidence = f
19960 .runner
19961 .state
19962 .change_summary
19963 .as_ref()
19964 .unwrap()
19965 .evidence
19966 .clone();
19967 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19968 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19969 assert!(posted(&mut f).await.body.contains("rationale updated"));
19970 assert_ne!(
19971 f.runner.state.change_summary.as_ref().unwrap().evidence,
19972 evidence
19973 );
19974 f.runner
19976 .state
19977 .change_summary
19978 .as_mut()
19979 .unwrap()
19980 .evidence
19981 .clear();
19982 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19983 assert!(posted(&mut f).await.body.contains("current specification"));
19984 }
19985
19986 #[tokio::test]
19987 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19988 let mut f = summary_fixture();
19989 let m = posted(&mut f).await;
19990 assert_eq!(m.title, "feat(own): add the own file");
19991 assert!(
19992 m.body
19993 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19994 "{}",
19995 m.body
19996 );
19997 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19998 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19999 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
20000 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
20001 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
20002 let kept = f.runner.state.change_summary.as_ref().unwrap();
20003 assert_eq!(
20004 (kept.head.as_str(), kept.base.as_str()),
20005 (head.as_str(), base.as_str())
20006 );
20007 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
20008 }
20009
20010 #[tokio::test]
20011 async fn review_only_summary_receives_the_configured_github_language() {
20012 let mut f = summary_fixture();
20013 f.runner.state.config.graph.github_language = "ja".into();
20014 let captured = f.repo.join("summary-prompt.txt");
20015 let author = &mut f.runner.state.config.agents[0];
20016 author.command = vec![
20017 "sh".into(),
20018 "-c".into(),
20019 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
20020 ];
20021 author.env.insert(
20022 "PROMPT_FILE".into(),
20023 captured.to_string_lossy().into_owned(),
20024 );
20025 std::fs::write(
20026 &f.reply,
20027 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
20028 )
20029 .unwrap();
20030 let m = posted(&mut f).await;
20031 let prompt = std::fs::read_to_string(captured).unwrap();
20032 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
20033 assert!(!prompt.contains("in English"));
20034 assert_eq!(m.title, "feat(own): ファイルを追加");
20035 assert!(m.body.contains("own.txt を追加します。"));
20036 }
20037
20038 #[tokio::test]
20039 async fn review_fixes_come_after_the_whole_change_summary() {
20040 let mut f = summary_fixture();
20041 f.runner.state.reviews = vec![
20042 round_with_notes(1, Some("handled the PATH case")),
20043 round_with_notes(2, None),
20044 ];
20045 let m = posted(&mut f).await;
20046 let summary = m.body.find("## Summary").expect("summary");
20047 let fixes = m.body.find("## Review rounds").expect("review rounds");
20048 assert!(summary < fixes, "{}", m.body);
20049 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
20050 assert!(m.body.contains("## Final verification"), "{}", m.body);
20051 }
20052
20053 #[tokio::test]
20054 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
20055 let mut f = summary_fixture();
20056 let first = posted(&mut f).await;
20057 let first_head = f.runner.state.change_summary.clone().unwrap().head;
20058 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
20060 let again = posted(&mut f).await;
20061 assert_eq!((again.title, again.body), (first.title, first.body));
20062
20063 let g = |args: &[&str]| {
20064 let out = std::process::Command::new("git")
20065 .args(args)
20066 .current_dir(&f.repo)
20067 .quiet()
20068 .output()
20069 .expect("spawn git");
20070 assert!(out.status.success(), "git {args:?}");
20071 };
20072 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
20073 g(&["add", "-A"]);
20074 g(&["commit", "-q", "-m", "fix(own): handle more"]);
20075 let second = posted(&mut f).await;
20076 let kept = f.runner.state.change_summary.clone().unwrap();
20077 assert_ne!(kept.head, first_head);
20078 assert!(second.body.contains("Not read yet."), "{}", second.body);
20079 assert!(!second.body.contains("so the first summary is read"));
20080 assert_eq!(second.title, "feat(own): other");
20081 }
20082
20083 #[tokio::test]
20084 async fn an_unavailable_summary_is_stated_and_never_invented() {
20085 let mut f = summary_fixture();
20086 std::fs::write(&f.reply, "no title line here").unwrap();
20087 let m = posted(&mut f).await;
20088 assert!(
20089 m.body
20090 .contains("No description of what this change does could be generated"),
20091 "{}",
20092 m.body
20093 );
20094 assert!(m.body.contains("internal reports remain in local records"));
20095 assert!(f.runner.state.change_summary.is_none());
20096 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
20097 let winner = f.runner.state.candidates[0].clone();
20099 let preview = f
20100 .runner
20101 .guarded_pr_message(&winner, None, false)
20102 .await
20103 .unwrap()
20104 .unwrap();
20105 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
20106 }
20107
20108 #[test]
20109 fn pr_message_review_single_commit_uses_its_subject() {
20110 let state = review_state(&["feat(nats): per-role user"]);
20111 let m = pr_message(&state, 'A');
20112 assert_eq!(m.title, "feat(nats): per-role user");
20113 assert!(!m.body.contains("Review the work already"), "{}", m.body);
20114 assert!(m.body.contains("## Commits under review"), "{}", m.body);
20115 }
20116
20117 #[test]
20118 fn pr_message_review_multi_commit_takes_the_oldest() {
20119 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
20120 let m = pr_message(&state, 'A');
20121 assert_eq!(m.title, "feat: the change");
20122 for s in ["feat: the change", "fix: typo", "fix: again"] {
20123 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
20124 }
20125 }
20126
20127 #[test]
20128 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
20129 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
20130 let mut state = review_state(&[first, "fix: later fixup"]);
20131 state.candidates[0].branch = "機能/ログイン".to_owned();
20132 let m = pr_message(&state, 'A');
20133 assert!(
20134 m.title.starts_with("chore: land candidate A of run"),
20135 "{}",
20136 m.title
20137 );
20138 }
20139 }
20140
20141 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
20142 BranchFacts {
20143 commits: commits
20144 .iter()
20145 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
20146 .collect(),
20147 stat: stat.to_owned(),
20148 }
20149 }
20150
20151 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
20152 let mut r = review_round(true, 0, 1, 1, true, true);
20153 r.round = round;
20154 r.fix = notes.map(|n| FixRecord {
20155 agent: "fixer".to_owned(),
20156 addressed: Vec::new(),
20157 rejected: Vec::new(),
20158 notes: n.to_owned(),
20159 committed: true,
20160 failed: None,
20161 duration_ms: 0,
20162 continuation: None,
20163 });
20164 r
20165 }
20166
20167 #[test]
20168 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
20169 let state = review_state(&["ignored"]);
20170 let f = facts(
20171 &[
20172 (
20173 "fix(login): resolve PATH on macOS",
20174 "Login shells skip rc files.",
20175 ),
20176 ("fix: address review", ""),
20177 ],
20178 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
20179 );
20180 let m = pr_message_with(&state, 'A', Some(&f));
20181 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
20182 assert!(
20183 m.body
20184 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
20185 "{}",
20186 m.body
20187 );
20188 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
20189 assert!(m.body.contains("## Diff stat"), "{}", m.body);
20190 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
20191 for banned in [
20192 "Review the work already",
20193 "no task statement",
20194 "Original task",
20195 ] {
20196 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
20197 }
20198 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
20199 }
20200
20201 #[test]
20202 fn pr_message_review_truncates_a_huge_first_commit_body() {
20203 let state = review_state(&["ignored"]);
20204 let f = facts(
20205 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
20206 "s",
20207 );
20208 let m = pr_message_with(&state, 'A', Some(&f));
20209 assert!(m.body.len() < 30_000, "{}", m.body.len());
20210 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
20211 assert!(
20212 m.body.contains("- ... 1 more commit(s)"),
20213 "{}",
20214 m.body.len()
20215 );
20216 assert!(m.body.ends_with("magi:candidate-a\n"));
20217 }
20218
20219 #[test]
20220 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
20221 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
20222 assert_eq!(m.title, "feat: x");
20223 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
20224 assert!(!m.body.contains("Diff stat"), "{}", m.body);
20225 assert!(!m.body.contains("no task statement"), "{}", m.body);
20226 }
20227
20228 #[test]
20229 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
20230 let mut state = review_state(&["日本語の件名"]);
20231 state.candidates[0].branch = "fix/macos-login-path".to_owned();
20232 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
20233 state.candidates[0].branch = "機能/ログイン".to_owned();
20234 assert!(
20235 pr_message(&state, 'A')
20236 .title
20237 .starts_with("chore: land candidate A")
20238 );
20239 }
20240
20241 #[test]
20242 fn pr_message_review_fixes_survive_a_clean_final_round() {
20243 let mut state = review_state(&["feat: x"]);
20244 state.reviews = vec![
20245 round_with_notes(1, Some("handled the PATH case")),
20246 round_with_notes(2, None),
20247 ];
20248 let body = pr_message(&state, 'A').body;
20249 assert!(
20250 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
20251 "{body}"
20252 );
20253 assert!(body.contains("### Round 1"), "{body}");
20254
20255 state.reviews = vec![
20256 round_with_notes(1, Some("first fix")),
20257 round_with_notes(2, Some("")),
20258 round_with_notes(3, Some("second fix")),
20259 round_with_notes(4, None),
20260 ];
20261 let body = pr_message(&state, 'A').body;
20262 assert!(
20263 body.contains("### Round 1") && !body.contains("first fix"),
20264 "{body}"
20265 );
20266 assert!(
20267 body.contains("### Round 3") && !body.contains("second fix"),
20268 "{body}"
20269 );
20270 assert!(body.contains("### Round 2"), "{body}");
20271 }
20272
20273 const FINAL: &str = "abcdef1234567890";
20274
20275 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20276 let mut r = round_with_notes(round, None);
20277 r.verified_head = verified.map(str::to_owned);
20278 r
20279 }
20280
20281 fn ok_gate(state: &mut RunState, head: Option<&str>) {
20282 state.gate = vec![CommandOutcome {
20283 command: "gate".to_owned(),
20284 code: Some(0),
20285 output_tail: String::new(),
20286 duration_ms: 0,
20287 resource_blocked: false,
20288 }];
20289 state.gate_ran = true;
20290 state.gate_head = head.map(str::to_owned);
20291 }
20292
20293 fn body_at(state: &RunState) -> String {
20294 pr_message_raw(
20295 state,
20296 'A',
20297 None,
20298 crate::pr_outcome::FinalHead::At(FINAL),
20299 ChangeNote::NotAsked,
20300 )
20301 .body
20302 }
20303
20304 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20305
20306 #[test]
20307 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20308 let mut state = review_state(&["feat: x"]);
20309 state.reviews = vec![
20310 round_with_notes(1, Some(SANDBOXED)),
20311 passing_round(2, Some(FINAL)),
20312 ];
20313 ok_gate(&mut state, Some(FINAL));
20314 let body = body_at(&state);
20315 assert!(!body.contains("Handled the PATH case."), "{body}");
20316 assert!(!body.contains("could not commit"), "{body}");
20317 assert!(!body.contains("could not run"), "{body}");
20318 assert!(
20319 body.contains("The fix is committed on the branch"),
20320 "{body}"
20321 );
20322 assert!(!body.contains("sandbox"), "{body}");
20323 assert!(
20324 body.contains("Tests (`verify.e2e`): passed on the final commit"),
20325 "{body}"
20326 );
20327 assert!(
20328 body.contains("Gate (`verify.gate`): passed on the final commit"),
20329 "{body}"
20330 );
20331 assert!(
20333 state.reviews[0]
20334 .fix
20335 .as_ref()
20336 .unwrap()
20337 .notes
20338 .contains("could not commit")
20339 );
20340 }
20341
20342 #[test]
20343 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20344 let mut state = review_state(&["feat: x"]);
20345 let mut r1 = round_with_notes(1, Some(SANDBOXED));
20346 r1.fix.as_mut().unwrap().committed = false;
20347 let mut r2 = passing_round(2, Some(FINAL));
20348 r2.e2e[0].code = Some(1);
20349 state.reviews = vec![r1, r2];
20350 let body = body_at(&state);
20351 assert!(!body.contains("could not commit"), "{body}");
20352 assert!(!body.contains("could not run"), "{body}");
20353 assert!(body.contains("This round produced no commit"), "{body}");
20354 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20355 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20356 assert!(!body.contains("passed"), "{body}");
20357 }
20358
20359 #[test]
20360 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20361 let mut state = review_state(&["feat: x"]);
20362 state.reviews = vec![
20363 round_with_notes(1, Some(SANDBOXED)),
20364 passing_round(2, Some("0123456789abcdef")),
20365 ];
20366 ok_gate(&mut state, Some("0123456789abcdef"));
20367 let body = body_at(&state);
20368 assert!(!body.contains("HTTP tests could not run"), "{body}");
20369 assert!(!body.contains("could not commit"), "{body}");
20371 assert!(
20372 body.contains("passed on an earlier commit (`0123456`)"),
20373 "{body}"
20374 );
20375 assert!(!body.contains("passed on the final commit"), "{body}");
20376
20377 ok_gate(&mut state, None);
20379 let body = body_at(&state);
20380 assert!(
20381 body.contains("passed on a commit that was not recorded"),
20382 "{body}"
20383 );
20384 }
20385
20386 #[test]
20387 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20388 let state = state_with_summary(
20389 "add retries to the client",
20390 "TITLE: feat: retries\n\nDid it.",
20391 );
20392 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20393 assert_eq!(m.title, "feat: retries");
20394 assert!(
20395 m.body.contains("<summary>Original task</summary>"),
20396 "{}",
20397 m.body
20398 );
20399 assert!(!m.body.contains("Commits under review"), "{}", m.body);
20400 assert!(
20401 m.body
20402 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20403 "{}",
20404 m.body
20405 );
20406 }
20407
20408 #[test]
20409 fn pr_message_review_bounds_a_long_english_subject() {
20410 let long = format!("feat: {}", "word ".repeat(100));
20411 let m = pr_message(&review_state(&[&long]), 'A');
20412 assert!(m.title.starts_with("feat: word"), "{}", m.title);
20413 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20414 }
20415
20416 #[test]
20417 fn pr_message_implementation_run_is_unchanged_by_review_support() {
20418 let state = state_with_summary("add retries\n\ndetails", "- did some things");
20419 let m = pr_message(&state, 'A');
20420 assert_eq!(m.title, "add retries");
20421 assert!(m.body.contains("<summary>Original task</summary>"));
20422 assert!(!m.body.contains("Commits under review"));
20423 assert_eq!(landing_subject_source(&state), state.instruction);
20424 }
20425
20426 #[test]
20427 fn review_run_squash_subject_is_the_change_not_the_prompt() {
20428 let state = review_state(&["feat: the change", "fix: typo"]);
20429 let source = landing_subject_source(&state);
20430 assert_eq!(land::merge_subject("", &source), "feat: the change");
20431 assert_eq!(
20432 land::merge_subject("magi: candidate A (uncommitted work)", &source),
20433 "feat: the change"
20434 );
20435 assert_eq!(
20437 land::merge_subject("feat: renamed by hand", &source),
20438 "feat: renamed by hand"
20439 );
20440 let blank = review_state(&["日本語"]);
20441 assert!(
20442 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20443 );
20444 }
20445
20446 #[test]
20447 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20448 let state = review_state(&["feat: the change"]);
20449 let source = landing_subject_source(&state);
20450 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20451 assert_eq!(
20452 land::merge_subject(landing_title(&state, old), &source),
20453 "feat: the change"
20454 );
20455 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20456 let task = state_with_summary("add retries", "");
20457 assert_eq!(landing_title(&task, old), old);
20458 }
20459
20460 #[test]
20461 fn pr_message_describes_the_change_not_the_task() {
20462 let state = state_with_summary(
20463 "今回やってほしいこと: results projector を直す",
20464 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20465 );
20466 let m = pr_message(&state, 'A');
20467 assert_eq!(m.title, "fix(web): batch the runs list reads");
20468 assert!(
20469 m.body.starts_with("## Summary\n\n- reads run.json once"),
20470 "{}",
20471 m.body
20472 );
20473 assert!(!m.body.contains("TITLE:"), "{}", m.body);
20474 let task_at = m.body.find("今回やってほしいこと").unwrap();
20475 let details_at = m.body.find("<details>").unwrap();
20476 assert!(
20477 details_at < task_at,
20478 "the task lives inside <details>: {}",
20479 m.body
20480 );
20481 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20482 assert!(m.body.contains("magi:candidate-a"));
20483 }
20484
20485 #[test]
20486 fn pr_message_falls_back_to_the_task_without_a_title_line() {
20487 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20488 let m = pr_message(&state, 'A');
20489 assert_eq!(m.title, "add retries");
20490 assert!(
20491 m.body.contains("## Summary\n\n- did some things"),
20492 "{}",
20493 m.body
20494 );
20495
20496 let none = RunState::new(
20497 PathBuf::from("/repo"),
20498 "main".to_owned(),
20499 "abc1234".to_owned(),
20500 "add retries".to_owned(),
20501 Config::default(),
20502 );
20503 let m = pr_message(&none, 'A');
20504 assert_eq!(m.title, "add retries");
20505 assert!(!m.body.contains("## Summary"), "{}", m.body);
20506 }
20507
20508 #[test]
20509 fn pr_message_refuses_the_candidate_commit_subject() {
20510 for bad in [
20511 "TITLE: magi: candidate A (uncommitted work)",
20512 "TITLE: chore: stuff (uncommitted work)",
20513 "TITLE: ",
20514 ] {
20515 let state = state_with_summary("add retries", bad);
20516 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20517 }
20518 }
20519
20520 #[test]
20521 fn pr_message_bounds_a_very_long_task_and_title() {
20522 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20523 let state = state_with_summary(&long, "- nothing");
20524 let m = pr_message(&state, 'A');
20525 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20526 assert!(!m.title.contains('\n'));
20527
20528 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20529 let m = pr_message(&state, 'A');
20530 assert!(m.title.starts_with("feat: "));
20531 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20532 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20533 }
20534
20535 fn long_title_of(instruction: &str) -> String {
20536 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20537 }
20538
20539 #[test]
20540 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20541 let first = "Make the landing path keep a readable title for long tasks";
20542 let line = format!(
20543 "{first}. {}",
20544 "And then keep going with more words ".repeat(20)
20545 );
20546 let t = long_title_of(&line);
20547 assert_eq!(t, first);
20548 assert!(!t.starts_with("chore: land"));
20549 }
20550
20551 #[test]
20552 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20553 let line = "word ".repeat(200);
20554 let t = long_title_of(&line);
20555 assert!(t.ends_with("word..."), "{t}");
20556 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20557 }
20558
20559 #[test]
20560 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20561 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20562 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20563 }
20564 }
20565
20566 #[test]
20567 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20568 let mut state = state_with_summary("再試行を追加する", "- nothing");
20569 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20570 state.config.graph.github_language = "ja".into();
20571 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20572 }
20573
20574 #[test]
20575 fn pr_message_title_limit_is_exact() {
20576 let at = "a".repeat(PR_TITLE_MAX);
20577 assert_eq!(long_title_of(&at), at);
20578 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20579 assert!(over.ends_with("..."), "{over}");
20580 assert_eq!(over.chars().count(), PR_TITLE_MAX);
20581 }
20582
20583 #[test]
20584 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20585 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20586 let t = long_title_of(&line);
20587 assert!(t.ends_with("..."), "{t}");
20588 assert!(t.is_ascii(), "{t}");
20589 }
20590
20591 #[test]
20592 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20593 let line = format!(
20594 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20595 "plus more filler words ".repeat(20)
20596 );
20597 let t = long_title_of(&line);
20598 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20599 }
20600
20601 #[test]
20602 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20603 let line = format!("\n\n# {}", "title words ".repeat(40));
20604 let state = state_with_summary(&line, "- nothing");
20605 let m = pr_message(&state, 'A');
20606 assert!(m.title.starts_with("title words"), "{}", m.title);
20607 assert_eq!(
20608 land::merge_subject(&m.title, &landing_subject_source(&state)),
20609 m.title
20610 );
20611 assert_eq!(
20613 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20614 "feat: renamed by hand"
20615 );
20616 }
20617
20618 #[tokio::test]
20619 async fn github_text_rewrite_is_bounded_and_falls_back() {
20620 crate::run::pin_test_home();
20621 for (reply, accepted) in [
20622 (
20623 "TITLE: fix: retries\nAdd retries for failed requests.",
20624 true,
20625 ),
20626 (
20627 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20628 false,
20629 ),
20630 (
20631 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20632 true,
20633 ),
20634 (
20635 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20636 true,
20637 ),
20638 (
20639 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20640 true,
20641 ),
20642 (
20643 "TITLE: fix: retries\n日本語の説明です。 token=secret",
20644 false,
20645 ),
20646 ("TITLE: fix: retries\n", false),
20647 ] {
20648 let dir = tempfile::tempdir().unwrap();
20649 let mut runner = runner_at(RunStatus::Gating);
20650 runner.state = state_with_summary(
20651 "add retries",
20652 "TITLE: fix: retries\n日本語の説明を書きます。",
20653 );
20654 runner.state.repo = dir.path().to_owned();
20655 runner.state.candidates[0].worktree = dir.path().to_owned();
20656 let mut author = spec("alpha");
20657 author.command = vec![
20658 "sh".into(),
20659 "-c".into(),
20660 "printf '%s' \"$REWRITE_REPLY\"".into(),
20661 ];
20662 author.env.insert("REWRITE_REPLY".into(), reply.into());
20663 runner.state.config.agents = vec![author];
20664 seed_public_summary(&mut runner);
20665 let winner = runner.state.candidates[0].clone();
20666 let first = runner
20667 .guarded_pr_message(&winner, None, true)
20668 .await
20669 .unwrap();
20670 let message = if accepted {
20671 first.expect("a passing rewrite never asks")
20672 } else {
20673 assert!(first.is_none());
20675 assert!(runner.state.parked);
20676 let store = ask::Questions::open();
20677 let asked: Vec<_> = store
20678 .list()
20679 .into_iter()
20680 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20681 .collect();
20682 assert_eq!(asked.len(), 1);
20683 assert!(!asked[0].detail.contains("secret"));
20684 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20685 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20686 assert_eq!(
20687 asked[0]
20688 .choices
20689 .contains(&crate::github_text::USE_REWRITE.to_owned()),
20690 rewrite.is_some()
20691 );
20692 store
20695 .update(&asked[0].id, |q| {
20696 q.abandon("test");
20697 Ok(())
20698 })
20699 .unwrap();
20700 let again = runner
20701 .guarded_pr_message(&winner, None, true)
20702 .await
20703 .unwrap()
20704 .expect("an abandoned question falls back");
20705 let n = store
20706 .list()
20707 .into_iter()
20708 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20709 .count();
20710 assert_eq!(n, 1, "asked once per rejected text");
20711 again
20712 };
20713 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20714 assert!(!message.body.contains("secret"));
20715 assert!(!message.body.contains("prefix"));
20716 assert!(!message.title.contains("secret"));
20717 assert_eq!(
20718 message.body.contains("Add retries for failed requests."),
20719 accepted
20720 );
20721 assert_eq!(
20722 runner.state.seats["github-text-rewrite"].turns, 1,
20723 "only one rewrite invocation"
20724 );
20725 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20726 if accepted {
20727 assert!(runner.state.github_text.is_none());
20728 assert!(!runner.state.parked);
20729 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20730 assert!(message.body.contains("## Final verification"));
20731 assert!(message.body.contains("Evidence: final diff"));
20732 assert!(
20733 message
20734 .body
20735 .contains(&format!("magi:run/{}", runner.state.id))
20736 );
20737 }
20738 if !accepted {
20739 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20740 assert!(
20741 message
20742 .body
20743 .contains(&format!("magi:run/{}", runner.state.id))
20744 );
20745 }
20746 }
20747 }
20748
20749 fn scripted(id: &str, reply: &str) -> AgentSpec {
20750 let mut a = spec(id);
20751 a.command = vec![
20752 "sh".into(),
20753 "-c".into(),
20754 "printf '%s' \"$REWRITE_REPLY\"".into(),
20755 ];
20756 a.env.insert("REWRITE_REPLY".into(), reply.into());
20757 a
20758 }
20759
20760 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20761 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20762
20763 fn seed_public_summary(runner: &mut Runner) {
20765 let repo = &runner.state.repo;
20766 let g = |args: &[&str]| {
20767 let out = std::process::Command::new("git")
20768 .args(args)
20769 .current_dir(repo)
20770 .quiet()
20771 .output()
20772 .unwrap();
20773 assert!(out.status.success(), "git {args:?}");
20774 String::from_utf8_lossy(&out.stdout).trim().to_owned()
20775 };
20776 g(&["init", "-q", "-b", "main"]);
20777 g(&[
20778 "-c",
20779 "user.name=Test",
20780 "-c",
20781 "user.email=test@example.invalid",
20782 "commit",
20783 "-q",
20784 "--allow-empty",
20785 "-m",
20786 "feat: fixture",
20787 ]);
20788 let head = g(&["rev-parse", "HEAD"]);
20789 g(&["update-ref", "refs/remotes/origin/main", &head]);
20790 let evidence = crate::pr_outcome::summary_evidence(
20791 &runner.state,
20792 crate::pr_outcome::FinalHead::At(&head),
20793 );
20794 let generated = &runner.state.candidates[0].summary;
20795 runner.state.change_summary = Some(crate::run::ChangeSummary {
20796 base: head.clone(),
20797 head,
20798 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
20799 title: summary_title(generated).unwrap(),
20800 body: summary_without_title(generated),
20801 });
20802 }
20803
20804 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
20805 let mut runner = runner_at(RunStatus::Gating);
20806 runner.state = state_with_summary("add retries", JA_SUMMARY);
20807 if review_only {
20808 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
20809 }
20810 runner.state.repo = dir.to_owned();
20811 runner.state.candidates[0].worktree = dir.to_owned();
20812 runner.state.config.agents = agents;
20813 seed_public_summary(&mut runner);
20814 runner
20815 }
20816
20817 fn gate_questions(runner: &Runner) -> usize {
20818 ask::Questions::open()
20819 .list()
20820 .into_iter()
20821 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20822 .count()
20823 }
20824
20825 #[tokio::test]
20826 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
20827 crate::run::pin_test_home();
20828 let dir = tempfile::tempdir().unwrap();
20829 let mut runner = rewrite_runner(
20830 dir.path(),
20831 true,
20832 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20833 );
20834 runner.state.config.roles.language_judge =
20835 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
20836 let winner = runner.state.candidates[0].clone();
20837 let msg = runner
20838 .guarded_pr_message(&winner, None, true)
20839 .await
20840 .unwrap()
20841 .expect("an English rewrite posts without asking");
20842 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
20843 assert!(msg.body.contains("<details>kept</details>"));
20844 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
20845 assert_eq!(gate_questions(&runner), 0);
20846 assert!(runner.state.github_text.is_none());
20847 assert!(!runner.state.seats.contains_key("impl-A"));
20848 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20849 }
20850
20851 #[tokio::test]
20852 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
20853 crate::run::pin_test_home();
20854 let dir = tempfile::tempdir().unwrap();
20855 let mut runner = rewrite_runner(
20856 dir.path(),
20857 false,
20858 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20859 );
20860 runner.state.config.roles.language_judge =
20861 Some(crate::config::AgentChoice::One("beta".into()));
20862 runner.state.seats.insert(
20863 "impl-A".into(),
20864 SeatState::new("impl-A", "alpha", runner.state.seed),
20865 );
20866 let winner = runner.state.candidates[0].clone();
20867 let msg = runner
20868 .guarded_pr_message(&winner, None, true)
20869 .await
20870 .unwrap()
20871 .expect("rewritten");
20872 assert!(msg.body.contains("<details>kept</details>"));
20873 assert_eq!(runner.state.seats["impl-A"].turns, 0);
20874 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20875 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20876 }
20877
20878 #[tokio::test]
20879 async fn rewrite_reuses_a_resumable_implementer_session() {
20880 crate::run::pin_test_home();
20881 let dir = tempfile::tempdir().unwrap();
20882 let mut runner = rewrite_runner(
20883 dir.path(),
20884 false,
20885 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20886 );
20887 runner.state.config.roles.language_judge =
20888 Some(crate::config::AgentChoice::One("beta".into()));
20889 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20890 seat.turns = 1;
20891 runner.state.seats.insert("impl-A".into(), seat);
20892 let winner = runner.state.candidates[0].clone();
20893 runner
20894 .guarded_pr_message(&winner, None, true)
20895 .await
20896 .unwrap()
20897 .expect("rewritten");
20898 assert_eq!(runner.state.seats["impl-A"].turns, 2);
20899 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20900 }
20901
20902 #[tokio::test]
20903 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20904 crate::run::pin_test_home();
20907 let dir = tempfile::tempdir().unwrap();
20908 let mut exhausted = spec("alpha");
20909 exhausted.command = vec!["false".into()];
20910 let mut runner = rewrite_runner(
20911 dir.path(),
20912 false,
20913 vec![exhausted, scripted("beta", EN_REWRITE)],
20914 );
20915 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20917 "alpha".into(),
20918 "beta".into(),
20919 ]));
20920 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20921 seat.turns = 1;
20922 runner.state.seats.insert("impl-A".into(), seat);
20923 let winner = runner.state.candidates[0].clone();
20924 let msg = runner
20925 .guarded_pr_message(&winner, None, true)
20926 .await
20927 .unwrap()
20928 .expect("the role rewrites after the implementer fails");
20929 assert!(msg.body.contains("`src/net.rs`"));
20930 assert_eq!(
20931 runner.state.seats["impl-A"].turns, 2,
20932 "the implementer was tried once"
20933 );
20934 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20935 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20936 }
20937
20938 #[tokio::test]
20939 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20940 crate::run::pin_test_home();
20941 let dir = tempfile::tempdir().unwrap();
20942 let mut broken = spec("bad");
20943 broken.command = vec!["false".into()];
20944 let mut runner = rewrite_runner(
20945 dir.path(),
20946 true,
20947 vec![
20948 broken,
20949 scripted("noformat", "Add retries."),
20950 scripted("good", EN_REWRITE),
20951 ],
20952 );
20953 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20954 "bad".into(),
20955 "noformat".into(),
20956 "good".into(),
20957 ]));
20958 let winner = runner.state.candidates[0].clone();
20959 let msg = runner
20960 .guarded_pr_message(&winner, None, true)
20961 .await
20962 .unwrap()
20963 .expect("the third agent rewrites");
20964 assert!(msg.body.contains("`src/net.rs`"));
20965 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20966 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20967 }
20968
20969 #[tokio::test]
20970 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20971 crate::run::pin_test_home();
20972 for (agents, role, saved) in [
20974 (
20975 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20976 Some("beta"),
20977 true,
20978 ),
20979 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20980 (Vec::new(), None, false),
20981 ] {
20982 let dir = tempfile::tempdir().unwrap();
20983 let mut runner = rewrite_runner(dir.path(), true, agents);
20984 runner.state.config.roles.language_judge =
20985 role.map(|r| crate::config::AgentChoice::One(r.into()));
20986 let winner = runner.state.candidates[0].clone();
20987 let first = runner
20988 .guarded_pr_message(&winner, None, true)
20989 .await
20990 .unwrap();
20991 assert!(first.is_none(), "withheld text parks the run");
20992 assert!(runner.state.parked);
20993 assert_eq!(gate_questions(&runner), 1);
20994 let g = runner.state.github_text.as_ref().unwrap();
20995 assert_eq!(g.rewrite.is_some(), saved);
20996 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20997 }
20998 }
20999
21000 #[tokio::test]
21001 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
21002 crate::run::pin_test_home();
21003 let dir = tempfile::tempdir().unwrap();
21004 let mut runner = runner_at(RunStatus::Gating);
21005 runner.state =
21006 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
21007 runner.state.repo = dir.path().to_owned();
21008 runner.state.candidates[0].worktree = dir.path().to_owned();
21009 let mut author = spec("alpha");
21010 author.command = vec![
21011 "sh".into(),
21012 "-c".into(),
21013 "printf '%s' \"$REWRITE_REPLY\"".into(),
21014 ];
21015 author.env.insert(
21016 "REWRITE_REPLY".into(),
21017 format!(
21018 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
21019 "English sentence. ".repeat(600)
21020 ),
21021 );
21022 runner.state.config.agents = vec![author];
21023 seed_public_summary(&mut runner);
21024 let winner = runner.state.candidates[0].clone();
21025 assert!(
21026 runner
21027 .guarded_pr_message(&winner, None, true)
21028 .await
21029 .unwrap()
21030 .is_none()
21031 );
21032 let record = runner.state.github_text.clone().unwrap();
21033 let rewrite = record.rewrite.clone().unwrap();
21034 assert_eq!(rewrite.failed, ["body-language"]);
21035 assert!(
21036 rewrite
21037 .body
21038 .ends_with(&format!("magi:run/{}", runner.state.id))
21039 );
21040 let store = ask::Questions::open();
21041 let q = store.get(record.question.as_ref().unwrap()).unwrap();
21042 assert!(
21043 q.choices
21044 .contains(&crate::github_text::USE_REWRITE.to_owned())
21045 );
21046 for label in [
21047 "ORIGINAL",
21048 "FALLBACK",
21049 "REWRITE",
21050 "Rules still failed: body-language",
21051 "truncated",
21052 ] {
21053 assert!(q.detail.contains(label), "{label}");
21054 }
21055 let artifact = std::fs::read_to_string(crate::run::artifact_path(
21056 &runner.state,
21057 &crate::github_text::artifact_name(&record.fingerprint),
21058 ))
21059 .unwrap();
21060 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
21061 runner.state =
21063 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
21064 store
21065 .update(&q.id, |q| {
21066 q.say("まだ日本語")?;
21067 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
21068 })
21069 .unwrap();
21070 assert!(
21071 runner
21072 .guarded_pr_message(&winner, None, true)
21073 .await
21074 .unwrap()
21075 .is_none()
21076 );
21077 let retry = runner.state.github_text.as_ref().unwrap();
21078 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
21079 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
21080 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
21081 assert!(
21082 q.posting_comparison
21083 .as_ref()
21084 .unwrap()
21085 .matches_detail(&q.detail)
21086 );
21087 store
21088 .update(&q.id, |q| {
21089 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21090 })
21091 .unwrap();
21092 let message = runner
21093 .guarded_pr_message(&winner, None, true)
21094 .await
21095 .unwrap()
21096 .unwrap();
21097 assert_eq!(message.title, rewrite.title);
21098 assert_eq!(message.body, rewrite.body);
21099 assert_eq!(
21100 runner.state.github_text.as_ref().unwrap().choice,
21101 Some(GithubTextChoice::Rewrite)
21102 );
21103 let again = runner
21104 .guarded_pr_message(&winner, None, true)
21105 .await
21106 .unwrap()
21107 .unwrap();
21108 assert_eq!(again.title, message.title);
21109 assert_eq!(again.body, message.body);
21110 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
21111 assert!(
21112 !runner
21113 .state
21114 .events
21115 .iter()
21116 .any(|e| e.message.contains(&rewrite.title)
21117 || e.message.contains("English sentence."))
21118 );
21119 }
21120
21121 #[tokio::test]
21122 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
21123 crate::run::pin_test_home();
21124 let dir = tempfile::tempdir().unwrap();
21125 let mut runner = runner_at(RunStatus::Gating);
21126 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
21127 runner.state.repo = dir.path().to_owned();
21128 runner.state.candidates[0].worktree = dir.path().to_owned();
21129 seed_public_summary(&mut runner);
21130 let winner = runner.state.candidates[0].clone();
21131 assert!(
21132 runner
21133 .guarded_pr_message(&winner, None, true)
21134 .await
21135 .unwrap()
21136 .is_none()
21137 );
21138 let record = runner.state.github_text.as_ref().unwrap();
21139 assert!(record.rewrite.is_none());
21140 let mut old = serde_json::to_value(record).unwrap();
21142 old.as_object_mut().unwrap().remove("rewrite");
21143 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
21144 assert!(old.rewrite.is_none());
21145 let id = record.question.clone().unwrap();
21146 ask::Questions::open()
21147 .update(&id, |q| {
21148 q.choices.push(crate::github_text::USE_REWRITE.into());
21149 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
21150 })
21151 .unwrap();
21152 let message = runner
21153 .guarded_pr_message(&winner, None, true)
21154 .await
21155 .unwrap()
21156 .unwrap();
21157 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
21158 assert_eq!(
21159 runner.state.github_text.as_ref().unwrap().choice,
21160 Some(GithubTextChoice::Fallback)
21161 );
21162 }
21163
21164 #[tokio::test]
21165 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
21166 crate::run::pin_test_home();
21167 let dir = tempfile::tempdir().unwrap();
21168 let mut runner = runner_at(RunStatus::Gating);
21169 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
21170 runner.state.repo = dir.path().to_owned();
21171 runner.state.candidates[0].worktree = dir.path().to_owned();
21172 seed_public_summary(&mut runner);
21173 let winner = runner.state.candidates[0].clone();
21174 assert!(
21175 runner
21176 .guarded_pr_message(&winner, None, true)
21177 .await
21178 .unwrap()
21179 .is_none()
21180 );
21181 let store = ask::Questions::open();
21182 let find = |run: &str| {
21183 store
21184 .list()
21185 .into_iter()
21186 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
21187 .unwrap()
21188 };
21189 let run = runner.state.id.clone();
21190 let q = find(&run);
21191 let fp = runner
21192 .state
21193 .github_text
21194 .as_ref()
21195 .unwrap()
21196 .fingerprint
21197 .clone();
21198 let name = crate::github_text::artifact_name(&fp);
21199 let path = crate::run::artifact_path(&runner.state, &name);
21200 assert!(q.detail.contains(&path.display().to_string()));
21201 assert!(q.detail.contains("title-language: title"));
21202 let saved = std::fs::read_to_string(&path).expect("artifact written");
21203 assert!(saved.contains("日本語のタイトル"));
21204 assert!(q.cwd.is_none());
21205 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
21206 let say = |id: &str, text: &str, choice: &str| {
21208 store
21209 .update(id, |q| {
21210 q.thread.push(ask::Turn {
21211 who: ask::Who::Operator,
21212 body: text.to_owned(),
21213 at: jiff::Timestamp::now(),
21214 note: None,
21215 });
21216 q.answer(ask::Answer::Choice(choice.to_owned()))
21217 })
21218 .unwrap();
21219 };
21220 say(&q.id, "まだ日本語", "use my text");
21221 assert!(
21222 runner
21223 .guarded_pr_message(&winner, None, true)
21224 .await
21225 .unwrap()
21226 .is_none()
21227 );
21228 let second = find(&run);
21229 assert_ne!(second.id, q.id);
21230 assert!(second.detail.contains("did not pass"));
21231 assert!(second.detail.contains("日本語のタイトル"));
21234 assert!(!second.detail.contains("まだ日本語"));
21235 assert!(second.detail.contains(&path.display().to_string()));
21236 say(&second.id, "fix: retry failed requests", "use my text");
21237 let message = runner
21238 .guarded_pr_message(&winner, None, true)
21239 .await
21240 .unwrap()
21241 .expect("a vetted replacement is posted");
21242 assert_eq!(message.title, "fix: retry failed requests");
21243 assert_eq!(
21244 runner
21245 .state
21246 .github_text
21247 .as_ref()
21248 .unwrap()
21249 .chosen_title
21250 .as_deref(),
21251 Some("fix: retry failed requests")
21252 );
21253 assert!(runner.state.github_text.as_ref().unwrap().resolved);
21254 let again = runner
21256 .guarded_pr_message(&winner, None, true)
21257 .await
21258 .unwrap()
21259 .unwrap();
21260 assert_eq!(again.title, message.title);
21261 assert_eq!(
21262 store
21263 .list()
21264 .into_iter()
21265 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
21266 .count(),
21267 2
21268 );
21269 }
21270
21271 #[test]
21272 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21273 let mut state = state_with_summary(
21277 "add retries",
21278 "TITLE: fix(web): batch reads\n- reads run.json once",
21279 );
21280 state.config.graph.language = "ja".to_owned();
21281 let m = pr_message(&state, 'A');
21282 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21283 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21284
21285 let task = "今回やってほしいこと: results projector を直す";
21288 let mut state = state_with_summary(task, "- no title line");
21289 state.config.graph.language = "ja".to_owned();
21290 let m = pr_message(&state, 'A');
21291 assert_eq!(
21292 m.title,
21293 format!("chore: land candidate A of run {}", state.id)
21294 );
21295 assert!(
21296 m.body.contains(&format!(
21297 "<summary>Original task</summary>\n\n{task}\n\n</details>"
21298 )),
21299 "{}",
21300 m.body
21301 );
21302 }
21303
21304 #[test]
21305 fn pr_message_scrubs_home_paths_and_addresses() {
21306 let state = state_with_summary(
21307 "fix it in /Users/someone/src/x",
21308 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21309 );
21310 let m = pr_message(&state, 'A');
21311 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21312 assert!(!m.body.contains(leak), "{}", m.body);
21313 }
21314 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21315 }
21316
21317 #[test]
21318 fn pr_message_survives_a_task_that_closes_details() {
21319 let state = state_with_summary("a </details> b", "TITLE: fix: x");
21320 let m = pr_message(&state, 'A');
21321 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21322 }
21323
21324 #[test]
21325 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21326 let cmd = manual_merge_command(
21327 MergeStyle::Squash,
21328 Path::new("/repo"),
21329 "b",
21330 "fix: \"quoted\" $(x) `y`\n\nbody",
21331 );
21332 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21333 }
21334
21335 #[test]
21336 fn manual_merge_command_matches_the_configured_style() {
21337 let repo = Path::new("/repo");
21338 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21339
21340 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21341 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21342
21343 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21344 assert_eq!(
21345 squash,
21346 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21347 \"Merge magi run 0832 (candidate A)\""
21348 );
21349
21350 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21351 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21352 }
21353
21354 #[test]
21355 fn a_nudge_gets_a_quarter_of_the_budget() {
21356 assert_eq!(retry_budget(secs(1200), true), secs(300));
21358 assert_eq!(retry_budget(secs(3600), true), secs(900));
21359 }
21360
21361 #[test]
21362 fn a_resent_prompt_keeps_the_whole_budget() {
21363 assert_eq!(retry_budget(secs(1200), false), secs(1200));
21366 assert_eq!(retry_budget(secs(60), false), secs(60));
21367 }
21368
21369 #[test]
21370 fn the_floor_never_exceeds_the_original_budget() {
21371 assert_eq!(retry_budget(secs(60), true), secs(60));
21375 assert_eq!(retry_budget(secs(480), true), secs(120));
21376 assert_eq!(retry_budget(secs(0), true), secs(0));
21377 }
21378
21379 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21380 agent::CommandEvidence {
21381 id: "item1".to_owned(),
21382 description: "cargo test".to_owned(),
21383 exit_code,
21384 result_summary: String::new(),
21385 source: "codex".to_owned(),
21386 }
21387 }
21388
21389 #[test]
21390 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21391 assert!(!has_unconfirmed_command(&[]));
21395 }
21396
21397 #[test]
21398 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21399 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21403 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21404 assert!(!has_unconfirmed_command(&[
21405 evidence(Some(0)),
21406 evidence(Some(101))
21407 ]));
21408 }
21409
21410 #[test]
21411 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21412 assert!(has_unconfirmed_command(&[
21413 evidence(Some(0)),
21414 evidence(None)
21415 ]));
21416 }
21417
21418 #[test]
21419 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21420 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21421 assert_eq!(
21422 verified_noop_claim(true, &[], text).as_deref(),
21423 Some("already fixed by b32cfc4, on main.")
21424 );
21425 }
21426
21427 #[test]
21428 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21429 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21432 assert!(verified_noop_claim(false, &[], text).is_none());
21433 }
21434
21435 #[test]
21436 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21437 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21438 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21439 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21441 }
21442
21443 #[test]
21444 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21445 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21446 }
21447
21448 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21451 runner.state.candidates = shape
21452 .iter()
21453 .enumerate()
21454 .map(|(i, &(empty, verified))| Candidate {
21455 index: i,
21456 label: (b'A' + i as u8) as char,
21457 agent: "sonnet".to_owned(),
21458 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21459 worktree: PathBuf::from(format!("/wt/{i}")),
21460 summary: String::new(),
21461 stat: String::new(),
21462 files: 0,
21463 commits: 0,
21464 empty,
21465 failed: None,
21466 verified_noop: verified.map(str::to_owned),
21467 duration_ms: 0,
21468 folded: false,
21469 })
21470 .collect();
21471 }
21472
21473 #[test]
21474 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21475 ask_test_home();
21476 let mut runner = runner_at(RunStatus::Implementing);
21477 set_candidates(
21478 &mut runner,
21479 &[
21480 (true, Some("already on main at b32cfc4")),
21481 (true, Some("same fix, see the existing test")),
21482 ],
21483 );
21484
21485 runner
21486 .after_implement()
21487 .expect("a verified no-op is not an error");
21488
21489 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21490 }
21491
21492 #[test]
21493 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21494 ask_test_home();
21495 let mut runner = runner_at(RunStatus::Implementing);
21496 set_candidates(
21500 &mut runner,
21501 &[(true, Some("already on main at b32cfc4")), (true, None)],
21502 );
21503
21504 let err = runner
21505 .after_implement()
21506 .expect_err("an unverified empty candidate must still fail the run");
21507
21508 assert!(
21509 err.to_string().contains("no candidate produced a change"),
21510 "{err}"
21511 );
21512 assert_eq!(runner.state.status, RunStatus::Failed);
21513 }
21514
21515 #[test]
21516 fn after_implement_still_fails_an_ordinary_all_empty_run() {
21517 ask_test_home();
21518 let mut runner = runner_at(RunStatus::Implementing);
21519 set_candidates(&mut runner, &[(true, None), (true, None)]);
21520
21521 let err = runner
21522 .after_implement()
21523 .expect_err("no candidate declared anything; this is an ordinary failure");
21524
21525 assert!(
21526 err.to_string().contains("no candidate produced a change"),
21527 "{err}"
21528 );
21529 assert_eq!(runner.state.status, RunStatus::Failed);
21530 }
21531}
21532
21533#[derive(Debug)]
21536struct GraftRestoreFailed(String);
21537
21538impl std::fmt::Display for GraftRestoreFailed {
21539 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21540 write!(
21541 f,
21542 "could not restore the winner to its pre-graft commit: {}",
21543 self.0
21544 )
21545 }
21546}
21547
21548impl std::error::Error for GraftRestoreFailed {}
21549
21550#[cfg(test)]
21554mod resource_wait_tests {
21555 use super::*;
21556 use std::collections::BTreeMap;
21557
21558 fn sh(id: &str, script: &str) -> AgentSpec {
21559 AgentSpec {
21560 id: id.to_owned(),
21561 kind: crate::config::AgentKind::Command,
21562 model: None,
21563 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21564 extra_args: Vec::new(),
21565 env: BTreeMap::new(),
21566 prompt_delivery: None,
21567 max_turn_secs: None,
21568 }
21569 }
21570
21571 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21572 SeatJob {
21573 seat: SeatState::new("impl-A", &spec.id, 1),
21574 spec,
21575 cwd: dir.to_path_buf(),
21576 prompt: "do it".to_owned(),
21577 timeout,
21578 allow_write: true,
21579 sessions: false,
21580 artifacts: dir.join("art"),
21581 stem: "impl-A".to_owned(),
21582 handover: None,
21583 }
21584 }
21585
21586 fn state(resource_wait_secs: u64) -> RunState {
21587 let mut config = Config::default();
21588 config.graph.timeout_resource_wait = resource_wait_secs;
21589 RunState::new(
21590 PathBuf::from("/repo"),
21591 "main".to_owned(),
21592 "abc1234".to_owned(),
21593 "task".to_owned(),
21594 config,
21595 )
21596 }
21597
21598 fn hold(cache: &Path) -> crate::cache::Guard {
21599 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21600 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21601 crate::cache::AcquireOutcome::Acquired(g) => g,
21602 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21603 }
21604 }
21605
21606 async fn run(
21607 jobs: Vec<SeatJob>,
21608 sem: Arc<Semaphore>,
21609 cache: Option<&Path>,
21610 st: &mut RunState,
21611 ) -> Vec<(usize, SeatState, AgentOutcome)> {
21612 let prompts = Prompts::default();
21613 let ctx = WaveCtx {
21614 run: "r",
21615 node: "implement",
21616 prompts: &prompts,
21617 cache,
21618 round: None,
21619 carry_seats: false,
21620 pauses: None,
21621 };
21622 wave(jobs, sem, &ctx, st, 0).await
21623 }
21624
21625 #[tokio::test]
21626 async fn verification_cache_wait_preserves_the_command_budget() {
21627 crate::run::pin_test_home();
21628 let tmp = tempfile::tempdir().unwrap();
21629 let cache = tmp.path().join("target");
21630 let guard = hold(&cache);
21631 let release = tokio::spawn(async move {
21632 tokio::time::sleep(Duration::from_millis(100)).await;
21633 guard.release();
21634 });
21635 let mut st = state(10);
21636 let budget = Duration::from_millis(50);
21637 let wait = repository_wait(&st, None);
21638 let (outcomes, _) = with_cache_lease(
21639 &mut st,
21640 Some(&cache),
21641 "gate",
21642 "gate",
21643 tmp.path(),
21644 "abc1234",
21645 budget,
21646 "test gate",
21647 &wait,
21648 |_, available| async move {
21649 assert_eq!(
21650 available, budget,
21651 "contention must not shorten verification"
21652 );
21653 (Vec::new(), false, Vec::new())
21654 },
21655 )
21656 .await
21657 .unwrap();
21658 release.await.unwrap();
21659 assert!(outcomes.is_empty(), "{outcomes:?}");
21660 assert!(
21661 st.events
21662 .iter()
21663 .any(|event| event.message.contains("waiting for"))
21664 );
21665 assert!(st.resource_wait.is_none());
21666 hold(&cache).release();
21668 }
21669
21670 #[tokio::test]
21671 async fn verification_cache_wait_expires_without_starting_commands() {
21672 crate::run::pin_test_home();
21673 let tmp = tempfile::tempdir().unwrap();
21674 let cache = tmp.path().join("target");
21675 let _held = hold(&cache);
21676 let mut st = state(0);
21677 let wait = repository_wait(&st, None);
21678 let (outcomes, _) = with_cache_lease(
21679 &mut st,
21680 Some(&cache),
21681 "gate",
21682 "gate",
21683 tmp.path(),
21684 "abc1234",
21685 Duration::from_secs(60),
21686 "test gate",
21687 &wait,
21688 |_, _| async { panic!("commands must not start without the cache lease") },
21689 )
21690 .await
21691 .unwrap();
21692 assert_eq!(outcomes.len(), 1);
21693 assert!(outcomes[0].resource_blocked);
21694 assert!(st.resource_wait.is_none());
21695 assert!(matches!(
21696 st.resource_block.unwrap().cause,
21697 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21698 ));
21699 }
21700
21701 #[tokio::test]
21702 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21703 crate::run::pin_test_home();
21704 let tmp = tempfile::tempdir().unwrap();
21705 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21706 let cache = tmp.path().join("cache");
21707 let mut state = state(21600);
21708 let wait = repository_wait(&state, None);
21709 let (outcomes, _) = with_cache_lease(
21710 &mut state,
21711 Some(&cache),
21712 "gate",
21713 "gate",
21714 tmp.path(),
21715 "head",
21716 Duration::from_secs(60),
21717 "gate",
21718 &wait,
21719 |_, _| async { panic!("freshness failure must not start verification") },
21720 )
21721 .await
21722 .unwrap();
21723 assert_eq!(outcomes.len(), 1);
21724 assert!(outcomes[0].resource_blocked);
21725 let block = state.resource_block.unwrap();
21726 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21727 assert!(!block.description().contains("timed out"));
21728 }
21729
21730 #[tokio::test]
21731 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21732 crate::run::pin_test_home();
21733 let tmp = tempfile::tempdir().unwrap();
21734 let cache = tmp.path().join("cache");
21735 let _held = hold(&cache);
21736 let mut state = state(21600);
21737 let pause = Pause::new();
21738 let interrupt = Pause::new();
21739 let cause = ParkCause::Operator;
21740 pause.park_for(cause.clone());
21741 let prompts = Prompts::default();
21742 let ctx = WaveCtx {
21743 run: "test",
21744 node: "fix",
21745 prompts: &prompts,
21746 cache: Some(&cache),
21747 round: None,
21748 carry_seats: false,
21749 pauses: Some((&pause, &interrupt)),
21750 };
21751 let result = wave(
21752 vec![job(
21753 sh("writer", "exit 99"),
21754 tmp.path(),
21755 Duration::from_secs(1),
21756 )],
21757 Arc::new(Semaphore::new(1)),
21758 &ctx,
21759 &mut state,
21760 0,
21761 )
21762 .await;
21763 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21764 assert_eq!(result[0].1.turns, 0);
21765 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21766 assert_eq!(
21767 state.resource_block.unwrap().cause,
21768 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21769 );
21770 assert!(state.active.is_empty());
21771 }
21772
21773 #[tokio::test]
21774 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21775 crate::run::pin_test_home();
21776 let tmp = tempfile::tempdir().unwrap();
21777 let cache = tmp.path().join("target-a");
21778 let guard = hold(&cache);
21779 tokio::spawn(async move {
21781 tokio::time::sleep(Duration::from_millis(1500)).await;
21782 guard.release();
21783 });
21784 let mut st = state(60);
21785 let results = run(
21786 vec![job(
21787 sh("alpha", "echo ok"),
21788 tmp.path(),
21789 Duration::from_secs(1),
21790 )],
21791 Arc::new(Semaphore::new(1)),
21792 Some(&cache),
21793 &mut st,
21794 )
21795 .await;
21796 assert!(
21797 matches!(results[0].2, AgentOutcome::Ok(_)),
21798 "the wait must not be charged to the agent"
21799 );
21800 assert!(st.resource_block.is_none());
21801 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21802 assert!(st.resource_wait.is_none() && st.active.is_empty());
21803 }
21804
21805 #[tokio::test]
21806 #[cfg(unix)]
21807 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
21808 use std::os::fd::AsRawFd as _;
21809 crate::run::pin_test_home();
21810 let tmp = tempfile::tempdir().unwrap();
21811 crate::git::git(tmp.path(), &["init"]).await.unwrap();
21812 let common = tmp.path().join(".git");
21813 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
21814 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
21815 assert_eq!(
21816 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
21817 0
21818 );
21819 let mut st = state(1);
21820 st.config.coordination = crate::config::test_coordination();
21821 let mut reader = job(
21822 sh("reader", "echo must-not-run"),
21823 tmp.path(),
21824 Duration::from_secs(30),
21825 );
21826 reader.allow_write = false;
21827 let results = run(
21828 vec![reader.clone()],
21829 Arc::new(Semaphore::new(1)),
21830 None,
21831 &mut st,
21832 )
21833 .await;
21834 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21835 assert_eq!(results[0].1.turns, 0);
21836 assert!(FailClass::of(&results[0].2).is_none());
21837 assert_eq!(
21838 st.resource_block.as_ref().unwrap().kind,
21839 "repository coordination"
21840 );
21841 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21842 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21843 drop(external);
21844 let editor = crate::repo_lock::SeatLease::for_checkout(
21845 tmp.path(),
21846 true,
21847 crate::config::test_coordination().as_ref(),
21848 )
21849 .await
21850 .unwrap();
21851 let pause = Pause::new();
21852 let interrupt = Pause::new();
21853 let prompts = Prompts::default();
21854 let ctx = WaveCtx {
21855 run: "r",
21856 node: "review",
21857 prompts: &prompts,
21858 cache: None,
21859 round: None,
21860 carry_seats: false,
21861 pauses: Some((&pause, &interrupt)),
21862 };
21863 let (results, ()) = tokio::join!(
21864 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
21865 async {
21866 tokio::time::sleep(Duration::from_millis(50)).await;
21867 pause.park();
21868 }
21869 );
21870 assert!(
21871 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
21872 );
21873 assert_eq!(results[0].1.turns, 0);
21874 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21875 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21876 drop(editor);
21877 }
21878
21879 #[tokio::test]
21880 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
21881 crate::run::pin_test_home();
21882 let tmp = tempfile::tempdir().unwrap();
21883 let cache = tmp.path().join("target-b");
21884 let _held = hold(&cache);
21885 let mut st = state(1);
21886 let results = run(
21887 vec![job(
21888 sh("alpha", "echo ok"),
21889 tmp.path(),
21890 Duration::from_secs(1),
21891 )],
21892 Arc::new(Semaphore::new(1)),
21893 Some(&cache),
21894 &mut st,
21895 )
21896 .await;
21897 let (_, seat, out) = &results[0];
21898 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21899 assert!(wave_blocked(&results));
21900 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21901 assert_eq!(seat.turns, 0, "nothing was asked");
21902 assert!(
21903 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21904 "no agent was spawned"
21905 );
21906 assert!(FailClass::of(out).is_none(), "not a failure class");
21907 let block = st.resource_block.as_ref().expect("recorded for the resume");
21908 assert_eq!(block.kind, "shared build cache");
21909 assert!(matches!(
21910 block.cause,
21911 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21912 ));
21913 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21914 assert!(st.active.is_empty());
21915 }
21916
21917 #[tokio::test]
21918 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21919 crate::run::pin_test_home();
21920 let tmp = tempfile::tempdir().unwrap();
21921 let mut st = state(1);
21922 let results = run(
21923 vec![job(
21924 sh("alpha", "echo ok"),
21925 tmp.path(),
21926 Duration::from_secs(1),
21927 )],
21928 Arc::new(Semaphore::new(0)),
21929 None,
21930 &mut st,
21931 )
21932 .await;
21933 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21934 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21935 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21936 }
21937
21938 #[tokio::test]
21939 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21940 crate::run::pin_test_home();
21941 let tmp = tempfile::tempdir().unwrap();
21942 let mut st = state(60);
21943 let results = run(
21944 vec![job(
21945 sh("alpha", "sleep 30"),
21946 tmp.path(),
21947 Duration::from_secs(1),
21948 )],
21949 Arc::new(Semaphore::new(1)),
21950 None,
21951 &mut st,
21952 )
21953 .await;
21954 let out = &results[0].2;
21955 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21956 let class = FailClass::of(out).expect("a failure");
21957 assert_eq!(class, FailClass::Timeout);
21958 assert!(should_hand_over(None, &class), "existing handover policy");
21959 assert!(st.resource_block.is_none());
21960 }
21961}