1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615impl Runner {
616 pub async fn start(
618 repo: &Path,
619 instruction: String,
620 config: Config,
621 origin: Origin,
622 ) -> Result<Self> {
623 Self::start_naming(repo, instruction, "", config, origin).await
624 }
625
626 pub async fn start_naming(
629 repo: &Path,
630 instruction: String,
631 _also_scan: &str,
632 config: Config,
633 origin: Origin,
634 ) -> Result<Self> {
635 let repo = git::toplevel(repo).await?;
636 let missing = agent::missing_programs(&config.agents);
637 if !missing.is_empty() {
638 bail!(
639 "these agent programs are not on PATH: {}. Fix the roster in \
640 magi.toml or install them.",
641 missing.join(", ")
642 );
643 }
644 let base_branch =
645 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646 .await?;
647 let fetched_base = resolve_base(
648 &repo,
649 &base_branch,
650 &config.merge.remote,
651 config.coordination.as_ref(),
652 )
653 .await?;
654 let base_commit = fetched_base.commit.clone();
655 let roles = config.resolve_roles()?;
656 let max_parallel = config.graph.max_parallel.max(1);
657 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658 state.fetched_base = Some(fetched_base);
662 state.origin = Some(origin);
663 state.reserve()?;
664 state.event("start", format!("run {} created", state.id));
665 state.save()?;
666 Ok(Self {
667 state,
668 roles,
669 sem: Arc::new(Semaphore::new(max_parallel)),
670 pause: Pause::new(),
671 interrupt: Pause::new(),
672 })
673 }
674
675 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689 Self::review_taking_over(repo, branch, config, None, origin).await
690 }
691
692 pub async fn review_taking_over(
699 repo: &Path,
700 branch: &str,
701 config: Config,
702 takeover: Option<crate::handover::Takeover>,
703 origin: Origin,
704 ) -> Result<Self> {
705 let repo = git::toplevel(repo).await?;
706 let missing = agent::missing_programs(&config.agents);
707 if !missing.is_empty() {
708 bail!(
709 "these agent programs are not on PATH: {}. Fix the roster in \
710 magi.toml or install them.",
711 missing.join(", ")
712 );
713 }
714 let base_branch =
715 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716 .await?;
717 if base_branch == branch {
718 bail!("`{branch}` is the base branch; there is nothing to review against");
719 }
720 let fetched_base = resolve_base(
721 &repo,
722 &base_branch,
723 &config.merge.remote,
724 config.coordination.as_ref(),
725 )
726 .await?;
727 let base_commit = fetched_base.commit.clone();
728
729 let roles = config.resolve_roles()?;
730 let max_parallel = config.graph.max_parallel.max(1);
731 let mut state = RunState::new(
732 repo.clone(),
733 base_branch,
734 base_commit.clone(),
735 String::new(),
736 config,
737 );
738 state.fetched_base = Some(fetched_base);
739 state.origin = Some(origin);
740 state.reserve()?;
741
742 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749 earlier: Vec::new(),
750 home: crate::run::home(),
751 choice: None,
752 task_instruction: None,
753 });
754 let wait = repository_wait(&state, None);
755 let coordination_config = state.config.coordination.clone();
756 let coordination =
757 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758 .await?;
759 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760 if let Some(released) = &released {
761 state.event(
762 "release",
763 format!(
764 "took `{branch}` over from run {}: its worktree was released: {}",
765 crate::run::short_of(&released.old_id),
766 released.audit
767 ),
768 );
769 }
770 state.foreign_checkout = crate::handover::preflight_review(
774 &repo,
775 branch,
776 &takeover.home,
777 &state.config.merge.remote,
778 )
779 .await?;
780 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
781 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
782 }
783 if let Some(choice) = takeover.choice.as_ref()
784 && let Err(e) =
785 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
786 .await
787 {
788 if let Some(released) = &released {
789 released.restore(&repo, branch).await;
790 }
791 return Err(e.context("applying the owner's answer about the diverged branch"));
792 }
793 for id in takeover.earlier.iter().rev() {
799 if let Ok(old) = RunState::load_under(id, &takeover.home)
800 && let Some(head) = old.pr_head.clone()
801 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
802 {
803 state.pr_head = Some(head);
804 break;
805 }
806 }
807 drop(coordination);
808 let opened = Self::open_review(
809 &repo,
810 branch,
811 state,
812 roles,
813 max_parallel,
814 base_commit,
815 &takeover,
816 )
817 .await;
818 if opened.is_err()
819 && let Some(released) = &released
820 {
821 let _coordination = crate::repo_lock::SeatLease::operation_with(
822 &repo,
823 coordination_config.as_ref(),
824 &wait,
825 )
826 .await?;
827 released.restore(&repo, branch).await;
828 }
829 opened
830 }
831
832 pub async fn recover_pr(
834 repo: &Path,
835 task: &str,
836 selection: crate::pr_recovery::Selection,
837 mut config: Config,
838 ) -> Result<Self> {
839 selection.configure(&mut config)?;
840 selection.check_task(task, &selection.instruction)?;
841 let base_branch =
842 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
843 .await?;
844 let fetched_base = resolve_base(
845 repo,
846 &base_branch,
847 &config.merge.remote,
848 config.coordination.as_ref(),
849 )
850 .await?;
851 let base_commit = fetched_base.commit.clone();
852 selection
853 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
854 .await?;
855 Box::pin(selection.verify_source(repo)).await?;
856 if let Some(proof) = &selection.source_proof {
857 anyhow::ensure!(
858 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
859 "source rebase base is outside the fetched landing base"
860 );
861 }
862 let roles = config.resolve_roles()?;
863 let max_parallel = config.graph.max_parallel.max(1);
864 let mut state = RunState::new(
865 repo.to_path_buf(),
866 base_branch,
867 base_commit.clone(),
868 selection.instruction.clone(),
869 config,
870 );
871 state.fetched_base = Some(fetched_base);
872 state.origin = Some(crate::run::Origin::queue(task));
873 state.pr_head = Some(selection.branch.clone());
874 state.pr_recovery = Some(selection.clone());
875 state.reserve()?;
876 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
877 state.discard_reserved();
879 return Err(e);
880 }
881 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
882 }
883
884 async fn open_selected(
885 repo: &Path,
886 state: RunState,
887 roles: ResolvedRoles,
888 max_parallel: usize,
889 base_commit: String,
890 selection: crate::pr_recovery::Selection,
891 ) -> Result<Self> {
892 let mut state = state;
893 let branch = state.branch_for('A');
894 let wait = repository_wait(&state, None);
895 let prepared = async {
896 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
897 git::git(repo, &["branch", &branch, selection.source()]).await?;
898 drop(coordination);
899 anyhow::Ok(())
900 }
901 .await;
902 if let Err(e) = prepared {
903 state.discard_reserved();
906 return Err(e);
907 }
908 state.event(
909 "recovery",
910 format!(
911 "selected implementation {} for {} (public lease {})",
912 selection.source(),
913 selection.url,
914 selection.head
915 ),
916 );
917 let takeover = crate::handover::Takeover {
918 earlier: Vec::new(),
919 home: crate::run::home(),
920 choice: None,
921 task_instruction: Some(selection.instruction.clone()),
922 };
923 if let Err(e) = state.save() {
924 state.discard_reserved();
925 return Err(e);
926 }
927 let id = state.id.clone();
928 let opened = Self::open_review(
929 repo,
930 &branch,
931 state,
932 roles,
933 max_parallel,
934 base_commit,
935 &takeover,
936 )
937 .await;
938 match opened {
939 Ok(runner) => Ok(runner),
940 Err(e) => {
941 if let Ok(mut partial) = RunState::load(&id) {
942 partial.status = RunStatus::Blocked;
943 partial.driver_exited = true;
944 partial.event(
945 "recovery",
946 format!("startup failed; retained source and candidate {branch}: {e:#}"),
947 );
948 partial.save()?;
949 }
950 Err(crate::pr_recovery::StartupFailed {
951 run: id,
952 cause: format!("{e:#}"),
953 }
954 .into())
955 }
956 }
957 }
958
959 async fn inherit(
964 repo: &Path,
965 branch: &str,
966 base_commit: &str,
967 takeover: &crate::handover::Takeover,
968 ) -> Option<crate::recovery::RecoveryRecord> {
969 use crate::recovery::{Relation, assess};
970 let newest = takeover.earlier.last()?;
971 let mut chosen = None;
972 for id in takeover.earlier.iter().rev() {
973 if let Ok(old) = RunState::load_under(id, &takeover.home)
974 && !old.reviews.is_empty()
975 && (old.winner().is_some_and(|w| w.branch == branch)
976 || old.pr_head.as_deref() == Some(branch))
977 {
978 chosen = Some(old);
979 break;
980 }
981 }
982 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
983 .await
984 .unwrap_or_default();
985 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
986 Some(last) if last.head == tip => Relation::Same,
987 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
988 _ => Relation::Diverged,
989 };
990 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
991 Some(assess(from, chosen.as_ref(), base_commit, relation))
992 }
993
994 async fn open_review(
997 repo: &Path,
998 branch: &str,
999 mut state: RunState,
1000 roles: ResolvedRoles,
1001 max_parallel: usize,
1002 base_commit: String,
1003 takeover: &crate::handover::Takeover,
1004 ) -> Result<Self> {
1005 let wait = repository_wait(&state, None);
1006 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1007 if state.foreign_checkout.is_some() {
1008 if state.pr_head.is_none() {
1010 state.pr_head = Some(branch.to_owned());
1011 }
1012 } else if state.pr_recovery.is_none() {
1013 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1014 }
1015 let start = review_base(
1018 repo,
1019 &state.config.merge.remote,
1020 &state.base_branch,
1021 &base_commit,
1022 branch,
1023 )
1024 .await;
1025 let log = git::log_oneline(repo, &start, branch)
1026 .await
1027 .unwrap_or_default();
1028 let log_text = if log.trim().is_empty() {
1029 "(no commit messages)"
1030 } else {
1031 log.trim()
1032 };
1033 let mut instruction = format!(
1034 "Review the work already on branch `{branch}`. There is no task \
1035 statement: what the change claims to do is whatever its commits \
1036 say.\n\n{log_text}"
1037 );
1038 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1042 if let Some(task) = takeover
1043 .task_instruction
1044 .as_deref()
1045 .filter(|t| !t.trim().is_empty())
1046 && rec.reviewed_head.is_some()
1047 {
1048 instruction = format!(
1049 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1050 task.trim()
1051 );
1052 rec.criteria_inherited = true;
1053 }
1054 state.event("recovery", rec.summary());
1055 state.recovery = Some(rec);
1056 }
1057 if let Some(selected) = &state.pr_recovery {
1058 instruction = selected.instruction.clone();
1059 }
1060 state.task_instruction = takeover.task_instruction.clone();
1061 state.instruction = instruction;
1062 state.reviewed_commits = Some(
1063 git::subjects(repo, &start, branch)
1064 .await
1065 .unwrap_or_default(),
1066 );
1067
1068 for id in &takeover.earlier {
1071 if let Ok(old) = RunState::load_under(id, &takeover.home)
1072 && old.repo == state.repo
1073 && (old.pr_head.as_deref() == Some(branch)
1074 || old.winner().is_some_and(|w| w.branch == branch))
1075 {
1076 for hold in old.branch_owner_holds {
1077 if !state.branch_owner_holds.contains(&hold) {
1078 state.event(
1079 "approval-policy",
1080 format!(
1081 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1082 hold.control.key(),
1083 hold.file,
1084 hold.head
1085 ),
1086 );
1087 state.branch_owner_holds.push(hold);
1088 }
1089 }
1090 }
1091 }
1092 let reviewed_head = git::rev_parse(repo, branch).await?;
1093 state.observe_branch_owner_holds(&reviewed_head).await;
1094
1095 let worktree = state.worktree_root().join("under-review");
1098 if let Some(parent) = worktree.parent() {
1099 tokio::fs::create_dir_all(parent).await.ok();
1100 }
1101 let path = worktree.to_string_lossy().to_string();
1102 let review_branch = if let Some(checkout) = &state.foreign_checkout {
1103 let shadow = format!("magi/{}/review", state.short());
1104 let tracking = format!("{}/{branch}", state.config.merge.remote);
1105 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1108 let mut head = checkout.synced_head.clone();
1109 if matches!(fetched, Ok(out) if out.ok()) {
1110 let remote = git::rev_parse(repo, &tracking).await?;
1111 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1112 head = remote;
1113 }
1114 }
1115 git::git(repo, &["branch", &shadow, &head]).await?;
1116 shadow
1117 } else {
1118 branch.to_owned()
1119 };
1120 git::git(repo, &["worktree", "add", &path, &review_branch])
1121 .await
1122 .with_context(|| {
1123 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1124 })?;
1125
1126 let setup = match ensure_setup_config(&mut state, repo).await {
1127 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1128 Err(e) => Err(e),
1129 };
1130 if let Err(e) = setup {
1131 git::worktree_remove_clean(repo, &worktree).await.ok();
1132 return Err(e);
1133 }
1134 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1135 .await
1136 .unwrap_or(0);
1137 if commits == 0 {
1138 git::worktree_remove_clean(repo, &worktree).await.ok();
1139 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1140 }
1141 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1142 .await
1143 .map(|f| f.len())
1144 .unwrap_or(0);
1145 if files == 0
1146 && let (Ok(head_tree), Ok(base_tree)) = (
1147 git::tree_of(&worktree, "HEAD").await,
1148 git::tree_of(&worktree, &base_commit).await,
1149 )
1150 && head_tree == base_tree
1151 {
1152 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1153 git::worktree_remove_clean(repo, &worktree).await.ok();
1154 bail!(
1155 "`{branch}` at {} has a tree identical to base {}; this usually means \
1156 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1157 against `{}/{branch}`) rather than an empty change",
1158 short(&head),
1159 short(&base_commit),
1160 state.config.merge.remote
1161 );
1162 }
1163 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1164 .await
1165 .unwrap_or_default();
1166
1167 state.candidates.push(Candidate {
1168 index: 0,
1169 label: 'A',
1170 agent: EXISTING_BRANCH.to_owned(),
1173 branch: review_branch,
1174 worktree,
1175 summary: String::new(),
1176 stat,
1177 files,
1178 commits,
1179 empty: false,
1180 failed: None,
1181 verified_noop: None,
1182 duration_ms: 0,
1183 folded: false,
1184 });
1185 state.tally = Some(Tally {
1186 first_choice: BTreeMap::from([('A', 0)]),
1187 borda: BTreeMap::new(),
1188 winner: 'A',
1189 rankings: 0,
1190 unanimous_initial: false,
1191 deliberated: false,
1192 changed_votes: 0,
1193 unanimous_final: false,
1194 tie_break: None,
1195 judges: 0,
1199 present: 0,
1200 quorum: 0,
1201 met_quorum: true,
1202 uncontested: Some("review-only run: nothing competed".to_owned()),
1203 });
1204 state.status = RunStatus::Reviewing;
1205 state.prep_complete = Some(true);
1206 state.event(
1207 "start",
1208 format!(
1209 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1210 state.id
1211 ),
1212 );
1213 state.save()?;
1214 Ok(Self {
1215 state,
1216 roles,
1217 sem: Arc::new(Semaphore::new(max_parallel)),
1218 pause: Pause::new(),
1219 interrupt: Pause::new(),
1220 })
1221 }
1222
1223 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1226 Self::authorize_base_recovery_request(home, id, task, false).await
1227 }
1228
1229 pub(crate) async fn authorize_base_recovery_from_resumption(
1231 home: &Path,
1232 id: &str,
1233 task: &str,
1234 ) -> Result<()> {
1235 Self::authorize_base_recovery_request(home, id, task, true).await
1236 }
1237
1238 async fn authorize_base_recovery_request(
1239 home: &Path,
1240 id: &str,
1241 task: &str,
1242 resumption: bool,
1243 ) -> Result<()> {
1244 let _owner = crate::pr_head::owner_lock(home, id)?;
1245 let mut state = RunState::load_under(id, home)?;
1246 crate::pr_head::ensure_owner(&state)?;
1247 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1248 anyhow::ensure!(
1249 queued.runs.last().is_some_and(|run| run == id)
1250 && queued.status != crate::queue::TaskStatus::Done
1251 && !queued.fresh_start
1252 && queued.review_branch.is_none(),
1253 "run task ownership changed"
1254 );
1255 anyhow::ensure!(
1256 state
1257 .origin
1258 .as_ref()
1259 .and_then(|o| o.task.as_deref())
1260 .is_none_or(|owner| owner == task),
1261 "run belongs to another task"
1262 );
1263 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1264 anyhow::ensure!(
1265 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1266 "run is live or its ownership is unknown"
1267 );
1268 let _execution = crate::driver::maintenance(home, id)?;
1271 anyhow::ensure!(
1272 state.status.resumable()
1273 && state.released_to.is_none()
1274 && state.retired_to.is_none()
1275 && state.pr_taken_over_by.is_none(),
1276 "run ownership is final"
1277 );
1278 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1279 anyhow::ensure!(
1280 !state.pr_publication_pending
1281 && !crate::handover::landing_checkpoint(&state)
1282 && state.merge_decision.is_none(),
1283 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1284 );
1285 let winner = state.winner().context("run has no retained winner")?;
1286 anyhow::ensure!(
1287 git::is_clean(&winner.worktree).await?,
1288 "retained worktree has uncommitted changes"
1289 );
1290 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1291 anyhow::ensure!(
1292 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1293 "retained worktree does not match the winner branch"
1294 );
1295 state.record_base_sync_stop(&head);
1296 state.save()?;
1297 if resumption {
1298 anyhow::ensure!(
1299 crate::daemon::compatible_run_instruction(&state, &queued),
1300 "resumption instruction does not match the retained run"
1301 );
1302 let approval = queued
1303 .resumption
1304 .as_ref()
1305 .context("no operator resumption")?;
1306 anyhow::ensure!(
1307 approval.instruction == queued.instruction,
1308 "resumption approval does not match the current task instruction"
1309 );
1310 state.authorize_base_sync_resumption(task, approval)?;
1311 } else {
1312 state.authorize_base_sync(task)?;
1313 }
1314 state.event(
1315 "base-recovery",
1316 "owner authorized one additional base rebase; fresh verification required",
1317 );
1318 state.save()?;
1319 Ok(())
1320 }
1321
1322 pub fn resume(id: &str) -> Result<Self> {
1324 let state = RunState::load(id)?;
1325 crate::pr_head::ensure_owner(&state)?;
1326 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1327 bail!(
1328 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1329 );
1330 }
1331 if let Some(to) = &state.released_to {
1332 bail!(
1333 "run {} cannot be resumed: its worktree was released to run {}",
1334 state.short(),
1335 crate::run::short_of(to)
1336 );
1337 }
1338 let roles = state.config.resolve_roles()?;
1339 if let Some(id) = &state.config.graph.selected_implementer
1340 && state.candidates.iter().any(|c| c.agent != *id)
1341 {
1342 bail!(
1343 "replacement implementer pin does not match the saved candidates; task must stay held"
1344 );
1345 }
1346 let max_parallel = state.config.graph.max_parallel.max(1);
1347 Ok(Self {
1348 state,
1349 roles,
1350 sem: Arc::new(Semaphore::new(max_parallel)),
1351 pause: Pause::new(),
1352 interrupt: Pause::new(),
1353 })
1354 }
1355
1356 pub async fn execute(&mut self) -> Result<()> {
1363 if let Some(selection) = &self.state.pr_recovery
1364 && (selection.source_proof.is_some() || selection.source_run.is_some())
1365 {
1366 Box::pin(selection.verify_source(&self.state.repo)).await?;
1367 }
1368 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1371 || self.state.land_approval.is_some()
1372 || self.state.merge_decision.is_some()
1373 || !self.state.merge_repairs.is_empty()
1374 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1375 {
1376 let claim = FixClaim::acquire(&self.state.dir())?;
1377 let disk = RunState::load(&self.state.id)?;
1378 let checkpoint = |s: &RunState| {
1379 (
1380 s.merge_repairs.len(),
1381 s.merge_repairs
1382 .last()
1383 .map(|r| (r.question.clone(), r.phase)),
1384 s.land_approval
1385 .as_ref()
1386 .map(|a| (a.question.clone(), a.head.clone())),
1387 )
1388 };
1389 if checkpoint(&disk) != checkpoint(&self.state) {
1390 bail!(
1391 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1392 );
1393 }
1394 Some(claim)
1395 } else {
1396 None
1397 };
1398 let mut execution = self.claim_driver().await?;
1399 let runner = &mut *execution.runner;
1400 let wait = runner.repository_wait();
1401 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1404 let result = match result {
1405 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1406 record_repository_block(&mut runner.state, &e);
1407 runner.park_here()?;
1408 Ok(())
1409 }
1410 other => other,
1411 };
1412 execution.driver.finish(&mut runner.state).await?;
1413 let ended = if result.is_err() {
1414 Some({
1415 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1416 if let Err(error) = &result {
1417 notice.message = format!(
1418 "Run {} stopped with an error: {}",
1419 runner.state.id,
1420 error.to_string().lines().next().unwrap_or("unknown error")
1421 );
1422 }
1423 notice
1424 })
1425 } else {
1426 crate::notices::run_ended(&runner.state)
1427 };
1428 if let Some(notice) = ended {
1429 crate::notices::raise(notice);
1430 }
1431 result
1432 }
1433
1434 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1435 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1436 }
1437
1438 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1439 let wait = self.repository_wait();
1440 coordinate_operation(&mut self.state, cwd, &wait).await
1441 }
1442
1443 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1445 let owner = match crate::pr_head::owner_lock_wait(
1446 &crate::run::home(),
1447 &self.state.id,
1448 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1449 )
1450 .await
1451 {
1452 Ok(owner) => owner,
1453 Err(error) => {
1454 let detail = format!("worktree maintenance or ownership update: {error:#}");
1455 self.state.resource_block = Some(crate::run::ResourceBlock {
1459 cause: crate::run::ResourceBlockCause::Other,
1460 node: "start".into(),
1461 kind: "worktree maintenance".into(),
1462 detail: detail.clone(),
1463 at: Timestamp::now(),
1464 });
1465 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1466 }
1467 };
1468 let disk = RunState::load(&self.state.id)?;
1469 crate::pr_head::ensure_owner(&disk)?;
1470 let claimed =
1471 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1472 anyhow::ensure!(
1477 disk.liveness(claimed) != crate::run::Liveness::Live,
1478 "run already has a live driver; wait for its execution to end"
1479 );
1480 if let Some(to) = &disk.released_to {
1481 bail!(
1482 "run {} cannot continue: its worktree was released to run {}",
1483 self.state.short(),
1484 crate::run::short_of(to)
1485 );
1486 }
1487 anyhow::ensure!(
1488 serde_json::to_value(&disk.base_sync_recovery)?
1489 == serde_json::to_value(&self.state.base_sync_recovery)?,
1490 "base recovery changed since this run was loaded; reload the saved run"
1491 );
1492 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1493 let mut execution = DriverExecution {
1494 runner: self,
1495 driver,
1496 };
1497 let _owner = owner;
1500 let state = &mut execution.runner.state;
1501 state.parked = false;
1506 state.resource_wait = None;
1507 state.resource_block = None;
1508 state.clear_active();
1515 let pid = std::process::id();
1530 state.driver_pid = Some(pid);
1531 state.driver_started_at = crate::proc::process_started_at(pid);
1532 state.driver_exited = false;
1533 state.driver_claimed = Some(true);
1534 state.begin_pass();
1535 execution.driver.bind(state);
1536 state.save()?;
1537 execution.driver.record(state)?;
1538 Ok(execution)
1539 }
1540
1541 async fn execute_graph(&mut self) -> Result<()> {
1542 self.state.configure_recovery_resume()?;
1543 self.persist_candidates().await?;
1546 if self.state.base_sync_blocked() {
1549 self.sync_to_base().await?;
1550 if !self.state.base_sync_blocked()
1551 && self.state.status != RunStatus::AlreadyInBase
1552 && !self.resource_halt()?
1553 {
1554 self.finish_after_tally().await?;
1555 }
1556 return Ok(());
1557 }
1558 if self
1559 .state
1560 .merge_decision
1561 .as_ref()
1562 .is_some_and(|decision| !decision.closed)
1563 && self.state.status != RunStatus::Landing
1564 {
1565 if crate::merge_decision::resumable_hold(&self.state) {
1566 self.state.merge = Some(MergeOutcome {
1567 mode: MergeMode::Pr,
1568 ok: false,
1569 detail: self
1570 .state
1571 .merge_decision
1572 .as_ref()
1573 .unwrap()
1574 .target
1575 .pr
1576 .clone(),
1577 empty: false,
1578 });
1579 self.run_land().await?;
1580 self.settle_questions();
1581 return Ok(());
1582 }
1583 bail!(
1584 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1585 );
1586 }
1587 if self.state.config.graph.selected_implementer.is_some()
1588 && self.state.status == RunStatus::Blocked
1589 && self.state.tally.is_none()
1590 && self.state.candidate_preservation.is_empty()
1591 && self.state.interrupted_implementations.is_empty()
1592 {
1593 bail!(
1594 "selected implementation is held; explicitly release the owning task to retry it"
1595 );
1596 }
1597
1598 if self.state.merge_repairs.last().is_some_and(|r| {
1599 matches!(
1600 r.phase,
1601 crate::run::MergeRepairPhase::Requested
1602 | crate::run::MergeRepairPhase::Fixing
1603 | crate::run::MergeRepairPhase::Reviewing
1604 )
1605 }) {
1606 if self.repair_merge_review().await? {
1607 self.run_land().await?;
1608 }
1609 self.settle_questions();
1610 return Ok(());
1611 }
1612 if self.state.pr_publication_pending {
1615 self.merge().await?;
1616 if self.resource_halt()? {
1617 return Ok(());
1618 }
1619 self.state.save()?;
1620 self.settle_questions();
1621 return Ok(());
1622 }
1623 if self.state.status == RunStatus::Stalled {
1636 if self.recover_stall().await? {
1637 self.finish_after_tally().await?;
1638 } else {
1639 self.state.save()?;
1641 }
1642 return Ok(());
1643 }
1644 if self.state.status == RunStatus::Landing
1654 || crate::handover::landing_checkpoint(&self.state)
1655 {
1656 self.state.land_sync_pending = true;
1657 self.state.save()?;
1658 self.run_land().await?;
1659 self.settle_questions();
1664 return Ok(());
1665 }
1666 if self.state.github_text.is_some()
1671 && self.state.merge.is_none()
1672 && self.state.status == RunStatus::Gating
1673 {
1674 self.merge().await?;
1675 if self.resource_halt()? {
1676 return Ok(());
1677 }
1678 self.state.save()?;
1679 return Ok(());
1680 }
1681 if let Err(e) = self.prep().await {
1682 if e.is::<crate::repo_lock::ResourceBlocked>() {
1683 return Err(e);
1684 }
1685 self.state.status = RunStatus::Blocked;
1686 self.state
1687 .prep_failure
1688 .get_or_insert_with(|| format!("{e:#}"));
1689 self.state
1690 .event("prep", format!("preparation stopped: {e:#}"));
1691 self.state.save()?;
1692 return Err(e);
1693 }
1694 if self.park_here()? {
1695 return Ok(());
1696 }
1697 self.advise().await?;
1698 if self.park_here()? {
1699 return Ok(());
1700 }
1701 self.implement().await?;
1702 if self.park_here()? {
1703 return Ok(());
1704 }
1705 if self.state.status == RunStatus::VerifiedNoop {
1709 return Ok(());
1710 }
1711 self.judge().await?;
1712 if self.park_here()? {
1713 return Ok(());
1714 }
1715 self.deliberate().await?;
1716 if self.park_here()? {
1717 return Ok(());
1718 }
1719 self.vote().await?;
1720 if self.park_here()? {
1721 return Ok(());
1722 }
1723 self.tally()?;
1724 if self.state.status == RunStatus::Stalled {
1729 self.state.save()?;
1733 return Ok(());
1734 }
1735 self.finish_after_tally().await?;
1736 Ok(())
1737 }
1738
1739 fn stop_for_resource(&mut self) -> Result<()> {
1744 if self.pause.parked() || self.interrupt.parked() {
1747 self.park_here()?;
1748 return Ok(());
1749 }
1750 if self.state.resource_block.is_some() {
1751 self.state.status = RunStatus::Blocked;
1752 }
1753 self.state.save()
1754 }
1755
1756 fn resource_halt(&mut self) -> Result<bool> {
1758 if self.state.resource_block.is_none() {
1759 return Ok(false);
1760 }
1761 self.stop_for_resource()?;
1762 Ok(true)
1763 }
1764
1765 fn park_here(&mut self) -> Result<bool> {
1772 let parking = self.pause.parked() || self.interrupt.parked();
1773 if !parking && self.state.resource_block.is_some() {
1774 self.stop_for_resource()?;
1775 return Ok(true);
1776 }
1777 if !self.pause.parked() && !self.interrupt.parked() {
1782 return Ok(false);
1783 }
1784 if self.state.parked {
1785 return Ok(true);
1786 }
1787 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1788 if let Some(block) = &mut self.state.resource_block
1789 && matches!(
1790 block.cause,
1791 crate::run::ResourceBlockCause::Cancelled { .. }
1792 )
1793 {
1794 block.cause = crate::run::ResourceBlockCause::Cancelled {
1795 park: cause.clone(),
1796 };
1797 }
1798 let why = match cause.as_ref().map(ParkCause::label) {
1799 Some(reason) => format!(
1800 "parked after `{}` ({reason}) — resume to carry on from here",
1801 self.state.status.as_str()
1802 ),
1803 None => format!(
1804 "parked after `{}` — resume to carry on from here",
1805 self.state.status.as_str()
1806 ),
1807 };
1808 self.state.event("park", why);
1809 self.state
1810 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1811 self.state.parked = true;
1812 self.state.save()?;
1813 Ok(true)
1814 }
1815
1816 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1818 if self.state.prep_failure.is_none() {
1819 return Ok(());
1820 }
1821 let failed_at = self
1822 .state
1823 .events
1824 .iter()
1825 .rev()
1826 .find(|event| event.node == "prep")
1827 .map_or(self.state.updated_at, |event| event.at);
1828 anyhow::ensure!(
1829 approval_at > failed_at,
1830 "preparation retry requires a new explicit release after the failure"
1831 );
1832 self.state.prep_failure = None;
1833 self.state.prep_complete = Some(false);
1834 self.state
1835 .event("prep-retry", "operator approved retry of preparation");
1836 Ok(())
1837 }
1838
1839 pub fn on_pause(&mut self, pause: Pause) {
1841 self.pause = pause;
1842 }
1843
1844 pub fn watch_interrupt(&mut self, pause: Pause) {
1850 self.interrupt = pause;
1851 }
1852
1853 fn settle_questions(&mut self) {
1873 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1874 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1875 }
1876 }
1877
1878 async fn finish_after_tally(&mut self) -> Result<()> {
1881 self.graft().await?;
1884 if self.resource_halt()? {
1885 return Ok(());
1886 }
1887 self.fold_losers().await?;
1888 self.sync_to_base().await?;
1893 if self.hold_base_sync().await? || self.resource_halt()? {
1894 return Ok(());
1895 }
1896 if self.state.status == RunStatus::AlreadyInBase {
1897 return Ok(());
1898 }
1899 self.review_loop().await?;
1900 if self.resource_halt()? {
1901 return Ok(());
1902 }
1903 self.sync_to_base().await?;
1904 if self.hold_base_sync().await? || self.resource_halt()? {
1905 return Ok(());
1906 }
1907 if self.state.status == RunStatus::AlreadyInBase {
1908 return Ok(());
1909 }
1910 self.gate().await?;
1911 self.merge().await?;
1912 if self.resource_halt()? {
1913 return Ok(());
1914 }
1915 self.state.save()?;
1916 Ok(())
1917 }
1918
1919 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1926 let result = match ensure_setup_config(&mut self.state, repo).await {
1927 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1928 Err(e) => Err(e),
1929 };
1930 if let Err(e) = result {
1931 git::worktree_remove_clean(repo, wt).await.ok();
1932 if in_prep {
1933 self.rollback_prep(repo).await;
1934 }
1935 self.state.status = RunStatus::Blocked;
1936 self.state.event(
1937 "setup",
1938 format!("worktree setup failed in {}: {e:#}", wt.display()),
1939 );
1940 self.state.save()?;
1941 return Err(e);
1942 }
1943 Ok(())
1944 }
1945
1946 async fn rollback_prep(&mut self, repo: &Path) {
1948 let root = self.state.worktree_root();
1949 for c in std::mem::take(&mut self.state.candidates) {
1950 if matches!(
1951 git::worktree_remove_clean(repo, &c.worktree).await,
1952 Ok(true)
1953 ) {
1954 git::branch_delete(repo, &c.branch).await.ok();
1955 } else {
1956 self.state.candidates.push(c);
1957 }
1958 }
1959 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1960 for j in 1..=seats {
1961 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1962 .await
1963 .ok();
1964 }
1965 for k in 1..=self.state.config.graph.advisors {
1966 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1967 .await
1968 .ok();
1969 }
1970 }
1971
1972 async fn prep(&mut self) -> Result<()> {
1973 if let Some(reason) = &self.state.prep_failure {
1974 bail!(
1975 "candidate preparation previously failed: {reason}; correct the source before retrying"
1976 );
1977 }
1978 if self
1979 .state
1980 .prep_complete
1981 .unwrap_or(!self.state.candidates.is_empty())
1982 {
1983 return Ok(());
1984 }
1985 let repo = self.state.repo.clone();
1986 let _coordination = self.coordinate(&repo).await?;
1989 let base = self.state.base_commit.clone();
1990 let plan = self.state.sources.plan(&repo, &base).await?;
1991 self.rollback_prep(&repo).await;
1992 if !self.state.candidates.is_empty() {
1993 bail!(
1994 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1995 );
1996 }
1997 self.state.prep_complete = Some(false);
1998 self.state.status = RunStatus::Prep;
1999 self.state.save()?;
2000 let repo = self.state.repo.clone();
2001 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2002 let root = self.state.worktree_root();
2003 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2004
2005 let hooks_dir = self.state.dir().join("hooks");
2008 if self.state.config.blind.commit_msg_hook {
2009 std::fs::create_dir_all(&hooks_dir)
2010 .with_context(|| format!("create {}", hooks_dir.display()))?;
2011 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2012 let path = hooks_dir.join("commit-msg");
2013 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2014 make_executable(&path)?;
2015 git::acquire_worktree_config(&repo).await?;
2023 self.state.enabled_worktree_config = true;
2024 }
2025
2026 ensure_setup_config(&mut self.state, &repo).await?;
2029
2030 for (index, (spec, label)) in self
2031 .roles
2032 .implementers
2033 .clone()
2034 .into_iter()
2035 .zip(labels)
2036 .enumerate()
2037 {
2038 let branch = self.state.branch_for(label);
2039 let worktree = root.join(format!("cand-{label}"));
2040 self.state.event(
2044 "prep-intent",
2045 serde_json::json!({
2046 "branch": branch,
2047 "worktree": worktree,
2048 "start": start,
2049 "picks": plan.picks,
2050 })
2051 .to_string(),
2052 );
2053 self.state.save()?;
2054 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2055 if self.state.config.blind.commit_msg_hook {
2056 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2057 }
2058 git::local_exclude(&worktree, "/.magi/").await?;
2059 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2060 git::worktree_remove_clean(&repo, &worktree).await.ok();
2063 git::branch_delete(&repo, &branch).await.ok();
2064 self.rollback_prep(&repo).await;
2065 self.state.status = RunStatus::Blocked;
2066 self.state
2067 .event("prep", format!("worktree setup failed: {e:#}"));
2068 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2069 self.state.save()?;
2070 return Err(e);
2071 }
2072 self.state.candidates.push(Candidate {
2073 index,
2074 label,
2075 agent: spec.id.clone(),
2076 branch,
2077 worktree,
2078 summary: String::new(),
2079 stat: String::new(),
2080 files: 0,
2081 commits: 0,
2082 empty: false,
2083 failed: None,
2084 verified_noop: None,
2085 duration_ms: 0,
2086 folded: false,
2087 });
2088 self.state.save()?;
2089 for pick in &plan.picks {
2090 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2091 {
2092 self.state.status = RunStatus::Blocked;
2093 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2094 self.state
2095 .event("prep", format!("cannot apply referenced commit: {e}"));
2096 self.state.save()?;
2097 return Err(e);
2098 }
2099 }
2100 }
2101
2102 for j in 1..=self.roles.judges.len() {
2103 let wt = root.join(format!("judge-{j}"));
2104 if !wt.exists() {
2105 git::worktree_add_detached(&repo, &wt, &base).await?;
2106 self.setup_seat_worktree(&repo, &wt, true).await?;
2107 }
2108 }
2109
2110 if self.state.config.graph.advise {
2118 for k in 1..=self.state.config.graph.advisors {
2119 let wt = root.join(format!("advisor-{k}"));
2120 if !wt.exists() {
2121 git::worktree_add_detached(&repo, &wt, &base).await?;
2122 self.setup_seat_worktree(&repo, &wt, true).await?;
2123 }
2124 }
2125 }
2126
2127 let authors: Vec<&str> = self
2132 .roles
2133 .implementers
2134 .iter()
2135 .map(|a| a.id.as_str())
2136 .collect();
2137 let overlap: Vec<String> = self
2138 .roles
2139 .judges
2140 .iter()
2141 .enumerate()
2142 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2143 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2144 .collect();
2145 if !overlap.is_empty() {
2146 let note = format!(
2147 "{} also authored a candidate; blind, but the panel is less \
2148 independent than {} distinct agents would be",
2149 overlap.join(", "),
2150 self.roles.judges.len()
2151 );
2152 self.state.event("prep", note);
2153 }
2154
2155 self.state.event(
2156 "prep",
2157 format!(
2158 "{} candidates, {} judges, base {} ({})",
2159 self.state.candidates.len(),
2160 self.roles.judges.len(),
2161 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2162 self.state.base_branch
2163 ),
2164 );
2165 self.state.prep_complete = Some(true);
2166 self.state.status = RunStatus::Implementing;
2167 self.state.save()?;
2168 Ok(())
2169 }
2170
2171 async fn advise(&mut self) -> Result<()> {
2204 let implement_untouched = self
2205 .state
2206 .candidates
2207 .iter()
2208 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2209 if !self.state.config.graph.advise || self.state.advise_attempted {
2210 return Ok(());
2211 }
2212 if !implement_untouched {
2213 self.state.event(
2214 "advise",
2215 "skipping the design-deliberation stage: at least one \
2216 candidate already shows implementation progress, so this \
2217 run is past the point the stage exists to run before"
2218 .to_owned(),
2219 );
2220 self.state.advise_attempted = true;
2221 self.state.save()?;
2222 return Ok(());
2223 }
2224 let run_id = self.state.id.clone();
2225 let prompts = self.state.config.prompts.clone();
2226 let instruction = self.state.instruction.clone();
2227 let language = self.state.config.graph.language.clone();
2228 let root = self.state.worktree_root();
2229 let n = self.state.config.graph.advisors;
2230 let where_recorded = self.state.dir().join("run.json");
2231
2232 let seats = match self.state.config.advisors() {
2233 Ok(seats) if !seats.is_empty() => seats,
2234 Ok(_) => {
2235 self.state.event(
2236 "advise",
2237 format!(
2238 "[graph] advisors is 0; skipping the design-deliberation \
2239 stage and continuing without a synthesis brief (see {})",
2240 where_recorded.display()
2241 ),
2242 );
2243 self.state.advise_attempted = true;
2244 self.state.save()?;
2245 return Ok(());
2246 }
2247 Err(e) => {
2248 self.state.event(
2249 "advise",
2250 format!(
2251 "could not resolve advisor seats ({e:#}); continuing \
2252 without a design-deliberation brief (see {})",
2253 where_recorded.display()
2254 ),
2255 );
2256 self.state.advise_attempted = true;
2257 self.state.save()?;
2258 return Ok(());
2259 }
2260 };
2261
2262 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2263 let artifacts = agent::artifacts_dir(&self.state.dir());
2264 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2265
2266 let mut jobs = Vec::new();
2267 for (i, spec) in seats.iter().cloned().enumerate() {
2268 let seat_key = format!("advisor-{}", i + 1);
2269 let seat = self.seat(&seat_key, &spec.id);
2270 jobs.push(SeatJob {
2271 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2272 spec,
2273 seat,
2274 cwd: worktrees[i % worktrees.len()].clone(),
2275 timeout,
2276 allow_write: false,
2277 sessions: false,
2278 artifacts: artifacts.clone(),
2279 stem: seat_key,
2280 handover: None,
2281 });
2282 }
2283
2284 self.state.event(
2285 "advise",
2286 format!(
2287 "{} advisor seat(s) sketching a design in parallel",
2288 jobs.len()
2289 ),
2290 );
2291 let mut quota_losses = Vec::new();
2292 let cache = self.state.config.cache_dir();
2293 let ctx = WaveCtx {
2294 carry_seats: false,
2295 pauses: Some((&self.pause, &self.interrupt)),
2296 run: &run_id,
2297 node: "advise",
2298 prompts: &prompts,
2299 cache: cache.as_deref(),
2300 round: None,
2301 };
2302 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2303 let results = ask_json_wave::<Proposal>(
2304 jobs,
2305 Arc::clone(&self.sem),
2306 self.state.config.graph.retries,
2307 &advisor_roster,
2308 &ctx,
2309 &mut quota_losses,
2310 &mut self.state,
2311 &|p: &Proposal| p.validate(),
2312 )
2313 .await;
2314 if self.state.resource_block.is_some() {
2315 return self.stop_for_resource();
2316 }
2317 self.state.quota.extend(quota_losses);
2318
2319 let mut records = Vec::with_capacity(results.len());
2320 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2321 let agent_id = seat.agent.clone();
2322 self.state.seats.insert(seat.key.clone(), seat);
2323 match res {
2324 Ok((proposal, out)) => {
2325 self.state
2326 .event("advise", format!("advisor-{} proposed a design", i + 1));
2327 records.push(advise::AdvisorRecord::proposed(
2328 i + 1,
2329 agent_id,
2330 proposal,
2331 out.duration_ms,
2332 ));
2333 }
2334 Err(e) => {
2335 self.state.event(
2336 "advise",
2337 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2338 );
2339 records.push(advise::AdvisorRecord::failed(
2340 i + 1,
2341 agent_id,
2342 e.to_string(),
2343 ));
2344 }
2345 }
2346 }
2347
2348 let mut advice = advise::Advice {
2349 records,
2350 synthesis: None,
2351 };
2352 if advice.proposals().is_empty() {
2353 self.state.event(
2354 "advise",
2355 "no advisor produced a usable proposal; continuing without a \
2356 synthesis brief"
2357 .to_owned(),
2358 );
2359 } else {
2360 match self
2361 .synthesize_brief(
2362 &advice,
2363 &instruction,
2364 &language,
2365 &worktrees[0],
2366 &artifacts,
2367 &run_id,
2368 &prompts,
2369 cache.as_deref(),
2370 )
2371 .await
2372 {
2373 Ok(Some(text)) => {
2374 self.state.event(
2375 "advise",
2376 "synthesized a design brief for the implementer".to_owned(),
2377 );
2378 advice.synthesis = Some(text);
2379 }
2380 Ok(None) => {
2381 self.state.event(
2382 "advise",
2383 "the synthesis seat produced nothing usable; continuing \
2384 without a design brief"
2385 .to_owned(),
2386 );
2387 }
2388 Err(e) => {
2389 if e.is::<crate::repo_lock::ResourceBlocked>() {
2390 return Err(e);
2391 }
2392 self.state.event(
2393 "advise",
2394 format!("could not synthesize a design brief: {e:#}"),
2395 );
2396 }
2397 }
2398 }
2399 advise::apply_reflection(&mut advice);
2400
2401 self.state.advice = Some(advice);
2402 self.state.advise_attempted = true;
2403 self.state.save()?;
2404 Ok(())
2405 }
2406
2407 #[allow(clippy::too_many_arguments)]
2419 async fn synthesize_brief(
2420 &mut self,
2421 advice: &advise::Advice,
2422 instruction: &str,
2423 language: &str,
2424 cwd: &Path,
2425 artifacts: &Path,
2426 run_id: &str,
2427 prompts: &Prompts,
2428 cache: Option<&Path>,
2429 ) -> Result<Option<String>> {
2430 let chain = agent::pick_chain(
2431 &self.state.config.agents,
2432 self.state.config.roles.synthesizer.as_ref(),
2433 &agent::installed,
2434 "synthesizer",
2435 )?;
2436 let proposals = advice.proposals();
2437 let mut prompt = prompt::with_overlay(
2438 prompt::synthesize_brief(instruction, &proposals, language),
2439 prompts.overlay("advise"),
2440 );
2441 if cache.is_some() {
2442 prompt.push('\n');
2447 prompt.push_str(&prompt::build_cache_note("advise", false));
2448 }
2449 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2450 let mut last = None;
2455 for (n, spec) in chain.iter().enumerate() {
2456 if n > 0 {
2457 self.state
2458 .event("advise", format!("synthesis falling back to {}", spec.id));
2459 }
2460 let mut seat = self.seat("advise-synthesis", &spec.id);
2461 let wait = self.repository_wait();
2462 let outcome = wait
2463 .scope(agent::invoke(
2464 spec,
2465 &mut seat,
2466 &Invocation {
2467 coordination: self.state.config.coordination.as_ref(),
2468 fetched_base: None,
2469 cwd,
2470 prompt: &prompt,
2471 timeout,
2472 allow_write: false,
2473 sessions: false,
2474 artifacts,
2475 stem: &if n == 0 {
2476 "advise-synthesis".to_owned()
2477 } else {
2478 format!("advise-synthesis-{}", spec.id)
2479 },
2480 run: run_id,
2481 node: "advise",
2482 cache_dir: None,
2483 attachments: &[],
2484 writable: &[],
2485 },
2486 ))
2487 .await;
2488 if outcome.is_ok() {
2489 self.state.seats.insert(seat.key.clone(), seat);
2490 }
2491 let advance = agent::chain_advances(&outcome);
2492 last = Some(outcome);
2493 if !advance {
2494 break;
2495 }
2496 }
2497 let out = last.expect("a chain holds at least one agent")?;
2500 if !out.usable() {
2501 return Ok(None);
2502 }
2503 let text =
2504 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2505 Ok((!text.trim().is_empty()).then_some(text))
2506 }
2507
2508 async fn implement(&mut self) -> Result<()> {
2511 self.persist_candidates().await?;
2512 let run_id = self.state.id.clone();
2517 let prompts = self.state.config.prompts.clone();
2518 let todo: Vec<usize> = self
2519 .state
2520 .candidates
2521 .iter()
2522 .enumerate()
2523 .filter(|(_, c)| {
2524 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2525 && c.failed.is_none()
2526 && !c.empty
2527 })
2528 .map(|(i, _)| i)
2529 .collect();
2530 if todo.is_empty() {
2531 return self.after_implement();
2532 }
2533 self.state.status = RunStatus::Implementing;
2534
2535 let language = self.state.config.graph.language.clone();
2536 let github_language = self.state.config.graph.github_language.clone();
2537 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2538 let sessions = self.state.config.graph.sessions;
2539 let artifacts = agent::artifacts_dir(&self.state.dir());
2540 let brief = self
2544 .state
2545 .advice
2546 .as_ref()
2547 .and_then(|a| a.synthesis.as_deref())
2548 .map(str::to_owned);
2549 let attachments = self.state.attachments.clone();
2550
2551 let mut jobs = Vec::new();
2552 for &i in &todo {
2553 let (index, label, worktree) = {
2554 let c = &self.state.candidates[i];
2555 (c.index, c.label, c.worktree.clone())
2556 };
2557 let spec = self.roles.implementers[index].clone();
2558 let seat_key = format!("impl-{label}");
2559 let seat = self.seat(&seat_key, &spec.id);
2560 let instruction = seeded_instruction(&self.state);
2561 jobs.push(SeatJob {
2562 spec,
2563 seat,
2564 prompt: prompt::implement_in(
2565 &instruction,
2566 &worktree.to_string_lossy(),
2567 &language,
2568 &github_language,
2569 brief.as_deref(),
2570 &attachments,
2571 ),
2572 cwd: worktree,
2573 timeout,
2574 allow_write: true,
2575 sessions,
2576 artifacts: artifacts.clone(),
2577 stem: format!("impl-{label}"),
2578 handover: None,
2579 });
2580 }
2581
2582 self.state.event(
2583 "implement",
2584 format!("{} candidates in parallel", jobs.len()),
2585 );
2586 let mut sent = jobs.clone();
2592 let cache = self.state.config.cache_dir();
2593 let ctx = WaveCtx {
2594 carry_seats: false,
2595 pauses: Some((&self.pause, &self.interrupt)),
2596 run: &run_id,
2597 node: "implement",
2598 prompts: &prompts,
2599 cache: cache.as_deref(),
2600 round: None,
2601 };
2602 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2603 if !wave_blocked(&results) {
2604 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2605 .await;
2606 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2607 .await?;
2608 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2609 .await;
2610 }
2611 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2612
2613 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2614 && results
2615 .iter()
2616 .any(|(_, _, out)| FailClass::of(out).is_some());
2617 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2618 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2619 if self
2623 .state
2624 .candidate_preservation
2625 .contains_key(&self.state.candidates[i].label)
2626 {
2627 self.state.seats.insert(seat.key.clone(), seat);
2628 }
2629 continue;
2630 }
2631 let seat_key = seat.key.clone();
2632 let agent = seat.agent.clone();
2641 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2642 self.state.seats.insert(seat.key.clone(), seat);
2643 let label = self.state.candidates[i].label;
2644 self.state.interrupted_implementations.remove(&label);
2645
2646 let (summary, duration, failed, verified_claim) = match out {
2647 AgentOutcome::Ok(o) => {
2648 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2649 let failed = (!o.usable()).then(|| {
2650 if o.timed_out {
2651 "agent timed out".to_owned()
2652 } else {
2653 format!(
2654 "agent exited with {}",
2655 crate::notices::exit_status(o.exit_code)
2656 )
2657 }
2658 });
2659 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2660 (text, o.duration_ms, failed, verified_claim)
2661 }
2662 AgentOutcome::Dropped(o) => {
2668 let why = o
2669 .dropped
2670 .as_ref()
2671 .map(|d| d.why.as_str())
2672 .unwrap_or("the CLI ended the stream without delivering its answer");
2673 (
2674 String::new(),
2675 o.duration_ms,
2676 Some(format!("the CLI dropped the stream ({why})")),
2677 None,
2678 )
2679 }
2680 AgentOutcome::Quota(o) => {
2681 self.state.quota.push(QuotaLoss {
2682 seat: seat_key,
2683 node: "implement".to_owned(),
2684 at: Timestamp::now(),
2685 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2686 });
2687 (
2688 String::new(),
2689 o.duration_ms,
2690 Some("rate limited (quota); produced no change".to_owned()),
2691 None,
2692 )
2693 }
2694 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2695 (String::new(), 0, Some(e), None)
2696 }
2697 };
2698
2699 let c = &mut self.state.candidates[i];
2702 if !exhausted_the_fallback_chain {
2703 c.agent = agent;
2704 }
2705 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2706 c.duration_ms = duration;
2707 c.failed = failed;
2708 c.verified_noop = verified_claim;
2709 self.state
2710 .candidate_preservation
2711 .insert(label, "awaiting persistence".to_owned());
2712 }
2713 self.state.save()?;
2714 self.persist_candidates().await?;
2715
2716 if resource_blocked {
2717 return self.stop_for_resource();
2718 }
2719 if selected_failed {
2720 self.state.status = RunStatus::Blocked;
2721 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2722 self.state.save()?;
2723 bail!(
2724 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2725 );
2726 }
2727 self.after_implement()
2728 }
2729
2730 async fn persist_candidates(&mut self) -> Result<()> {
2732 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2733 for label in labels {
2734 let i = self
2735 .state
2736 .candidates
2737 .iter()
2738 .position(|c| c.label == label)
2739 .context("pending candidate is missing")?;
2740 let c = self.state.candidates[i].clone();
2741 let interrupted = self.state.interrupted_implementations.contains(&label);
2742 let result = async {
2743 let wait = crate::repo_lock::WaitPolicy::new(
2744 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2745 || false,
2746 );
2747 let _coordination =
2748 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2749 if let Err(e) = git::snapshot_uncommitted(
2752 &c.worktree,
2753 &self
2754 .state
2755 .dir()
2756 .join("artifacts")
2757 .join(format!("candidate-{label}-recovery")),
2758 )
2759 .await
2760 {
2761 self.state.event(
2762 "preserve",
2763 format!("candidate {label} snapshot incomplete: {e:#}"),
2764 );
2765 }
2766 let rescued = git::rescue_commit(
2767 &c.worktree,
2768 &format!("magi: candidate {label} (uncommitted work)"),
2769 )
2770 .await?;
2771 self.state.note_withheld("implement", &rescued.withheld);
2772 let base = &self.state.base_commit;
2773 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2774 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2775 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2776 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2777 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2778 if c.empty && commits > 0 && !patch.trim().is_empty()
2779 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2780 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2781 || self.state.tally.is_some())
2782 {
2783 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2787 }
2788 let c = &mut self.state.candidates[i];
2789 c.commits = commits;
2790 c.stat = stat;
2791 c.files = files;
2792 let no_changes = commits == 0 || patch.trim().is_empty();
2793 c.empty = no_changes && !interrupted;
2794 if interrupted && no_changes {
2795 c.failed = None;
2796 c.verified_noop = None;
2797 }
2798 if !c.empty {
2799 c.failed = None;
2800 c.verified_noop = None;
2801 }
2802 self.state.event(
2803 "preserve",
2804 format!("candidate {label}: {files} files, {commits} commits"),
2805 );
2806 Ok::<_, anyhow::Error>(())
2807 }
2808 .await;
2809 match result {
2810 Ok(()) => {
2811 self.state.candidate_preservation.remove(&label);
2812 if !interrupted || self.state.candidates[i].files > 0 {
2813 self.state.interrupted_implementations.remove(&label);
2814 }
2815 }
2816 Err(e) => {
2817 let reason = format!("candidate {label} preservation failed: {e:#}");
2818 self.state.event("preserve", &reason);
2819 self.state.candidate_preservation.insert(label, reason);
2820 }
2821 }
2822 if self.state.candidate_preservation.is_empty()
2823 && self.state.status == RunStatus::Blocked
2824 && self.state.candidates.iter().all(|c| c.failed.is_none())
2825 {
2826 self.state.status = RunStatus::Implementing;
2830 self.state.event(
2831 "preserve",
2832 "candidate persistence recovered; implementation can continue",
2833 );
2834 }
2835 self.state.save()?;
2836 }
2837 if !self.state.candidate_preservation.is_empty() {
2838 self.state.status = RunStatus::Blocked;
2839 self.state.save()?;
2840 bail!(
2841 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2842 );
2843 }
2844 Ok(())
2845 }
2846
2847 async fn resume_undelivered(
2875 &mut self,
2876 results: &mut [(usize, SeatState, AgentOutcome)],
2877 sent: &[SeatJob],
2878 prompts: &Prompts,
2879 run_id: &str,
2880 ) {
2881 for (wi, seat, out) in results.iter_mut() {
2882 let Some(dropped) = (match &*out {
2883 AgentOutcome::Dropped(o) => o.dropped.clone(),
2884 _ => None,
2885 }) else {
2886 continue;
2887 };
2888 let Some(job) = sent.get(*wi) else { continue };
2889 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2891 self.state.event(
2892 "implement",
2893 format!(
2894 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2895 work is in the tree",
2896 seat.key, dropped.output_tokens, dropped.why
2897 ),
2898 );
2899 continue;
2900 }
2901 if !has_context(&job.spec, seat, job.sessions) {
2909 self.state.event(
2910 "implement",
2911 format!(
2912 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2913 is no session left to resume",
2914 seat.key, dropped.output_tokens, dropped.why
2915 ),
2916 );
2917 continue;
2918 }
2919 self.state.event(
2920 "implement",
2921 format!(
2922 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2923 conversation",
2924 seat.key, dropped.output_tokens, dropped.why
2925 ),
2926 );
2927 let mut retry = job.clone();
2928 retry.seat = seat.clone();
2929 retry.prompt = prompt::resume_after_drop(&dropped.why);
2930 retry.timeout = retry_budget(job.timeout, true);
2931 retry.stem = format!("{}-resume", job.stem);
2932 let cache = self.state.config.cache_dir();
2933 let ctx = WaveCtx {
2934 carry_seats: false,
2935 pauses: Some((&self.pause, &self.interrupt)),
2936 run: run_id,
2937 node: "implement",
2938 prompts,
2939 cache: cache.as_deref(),
2940 round: None,
2941 };
2942 let (resumed_seat, resumed) =
2943 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2944 *seat = resumed_seat;
2945 *out = resumed;
2946 }
2947 }
2948
2949 async fn resume_seat_handovers(
3014 &mut self,
3015 results: &mut [(usize, SeatState, AgentOutcome)],
3016 sent: &mut [SeatJob],
3017 prompts: &Prompts,
3018 run_id: &str,
3019 ) -> Result<()> {
3020 let instruction = seeded_instruction(&self.state);
3021 let language = self.state.config.graph.language.clone();
3022 let github_language = self.state.config.graph.github_language.clone();
3023 let brief = self
3024 .state
3025 .advice
3026 .as_ref()
3027 .and_then(|a| a.synthesis.as_deref())
3028 .map(str::to_owned);
3029 let attachments = self.state.attachments.clone();
3030 for (wi, seat, out) in results.iter_mut() {
3031 let others: BTreeSet<String> = sent
3034 .iter()
3035 .enumerate()
3036 .filter(|(j, _)| j != wi)
3037 .map(|(_, j)| j.spec.id.clone())
3038 .collect();
3039 let Some(job) = sent.get_mut(*wi) else {
3040 continue;
3041 };
3042 let start = self
3047 .roles
3048 .implementer_roster
3049 .iter()
3050 .position(|s| s.id == job.spec.id)
3051 .unwrap_or(0);
3052 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3053 let mut fallback_attempt = 0usize;
3054 let mut prev: Option<FailClass> = None;
3055 while let Some(cur) = FailClass::of(&*out) {
3056 if !should_hand_over(prev.as_ref(), &cur) {
3057 break;
3058 }
3059 let Some(next) =
3060 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3061 .cloned()
3062 else {
3063 break;
3064 };
3065 tried.insert(next.id.clone());
3066 fallback_attempt += 1;
3067
3068 let coordination = self.coordinate(&job.cwd).await?;
3069 if let Ok(r) = git::rescue_commit(
3070 &job.cwd,
3071 &format!(
3072 "magi: candidate {} (uncommitted work before {} fallback)",
3073 seat.key,
3074 if cur == FailClass::Quota {
3075 "quota"
3076 } else {
3077 "handover"
3078 }
3079 ),
3080 )
3081 .await
3082 {
3083 self.state.note_withheld("implement", &r.withheld);
3084 }
3085 drop(coordination);
3086
3087 record_handover(
3088 &mut self.state,
3089 "implement",
3090 &seat.key,
3091 &seat.agent,
3092 &next.id,
3093 &cur,
3094 &fail_reason(&*out),
3095 );
3096 prev = Some(cur.clone());
3097
3098 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3099 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3100 job.spec = next.clone();
3108 let mut retry = job.clone();
3109 retry.seat = new_seat;
3110 retry.prompt = prompt::implement_in(
3111 &instruction,
3112 &job.cwd.to_string_lossy(),
3113 &language,
3114 &github_language,
3115 brief.as_deref(),
3116 &attachments,
3117 );
3118 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3119 let cache = self.state.config.cache_dir();
3120 let ctx = WaveCtx {
3121 carry_seats: false,
3122 pauses: Some((&self.pause, &self.interrupt)),
3123 run: run_id,
3124 node: "implement",
3125 prompts,
3126 cache: cache.as_deref(),
3127 round: None,
3128 };
3129 let (fallback_seat, fallback_out) = run_one(
3130 retry,
3131 Arc::clone(&self.sem),
3132 &ctx,
3133 &mut self.state,
3134 fallback_attempt,
3135 )
3136 .await;
3137 *seat = fallback_seat;
3138 *out = fallback_out;
3139 }
3140 }
3141 Ok(())
3142 }
3143
3144 async fn resume_unconfirmed_commands(
3168 &mut self,
3169 results: &mut [(usize, SeatState, AgentOutcome)],
3170 sent: &[SeatJob],
3171 prompts: &Prompts,
3172 run_id: &str,
3173 ) {
3174 for (wi, seat, out) in results.iter_mut() {
3175 let AgentOutcome::Ok(o) = &*out else {
3176 continue;
3177 };
3178 if !has_unconfirmed_command(&o.commands) {
3179 continue;
3180 }
3181 let Some(job) = sent.get(*wi) else { continue };
3182 if !has_context(&job.spec, seat, job.sessions) {
3183 self.state.event(
3184 "implement",
3185 format!(
3186 "{}: the reply named a command whose own CLI never confirmed the exit \
3187 status of, but there is no session left to resume",
3188 seat.key
3189 ),
3190 );
3191 continue;
3192 }
3193 self.state.event(
3194 "implement",
3195 format!(
3196 "{}: the reply named a command whose own CLI never confirmed the exit \
3197 status of; resuming the conversation",
3198 seat.key
3199 ),
3200 );
3201 let mut retry = job.clone();
3202 retry.seat = seat.clone();
3203 retry.prompt = prompt::resume_incomplete(
3204 "a command in your last reply had no confirmed exit status",
3205 );
3206 retry.timeout = retry_budget(job.timeout, true);
3207 retry.stem = format!("{}-confirm", job.stem);
3208 let cache = self.state.config.cache_dir();
3209 let ctx = WaveCtx {
3210 carry_seats: false,
3211 pauses: Some((&self.pause, &self.interrupt)),
3212 run: run_id,
3213 node: "implement",
3214 prompts,
3215 cache: cache.as_deref(),
3216 round: None,
3217 };
3218 let (resumed_seat, resumed) =
3219 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3220 *seat = resumed_seat;
3221 *out = resumed;
3222 }
3223 }
3224
3225 async fn continue_fix_report(
3246 &mut self,
3247 mut seat: SeatState,
3248 parse_err: String,
3249 job: &SeatJob,
3250 prompts: &Prompts,
3251 run_id: &str,
3252 round: usize,
3253 ) -> (
3254 SeatState,
3255 Option<FixReport>,
3256 Option<String>,
3257 ContinuationRecord,
3258 ) {
3259 let mut last_err = parse_err;
3260 let mut cumulative_wait_ms = 0u64;
3261 let mut attempts = 0usize;
3262 loop {
3263 if !has_context(&job.spec, &seat, job.sessions) {
3264 self.state.event(
3265 "fix",
3266 format!(
3267 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3268 session left to resume into"
3269 ),
3270 );
3271 let outcome = if attempts == 0 {
3272 ContinuationOutcome::NoSession
3273 } else {
3274 ContinuationOutcome::Exhausted
3275 };
3276 return (
3277 seat,
3278 None,
3279 Some(format!("unparsable fix report: {last_err}")),
3280 ContinuationRecord {
3281 attempts,
3282 cumulative_wait_ms,
3283 outcome,
3284 },
3285 );
3286 }
3287 if attempts >= MAX_FIX_CONTINUATIONS {
3288 self.state.event(
3289 "fix",
3290 format!(
3291 "round {round}: fixer's reply still had no adoption report after \
3292 {attempts} continuation(s) ({last_err}); giving up"
3293 ),
3294 );
3295 return (
3296 seat,
3297 None,
3298 Some(format!(
3299 "unparsable fix report after {attempts} continuation(s): {last_err}"
3300 )),
3301 ContinuationRecord {
3302 attempts,
3303 cumulative_wait_ms,
3304 outcome: ContinuationOutcome::Exhausted,
3305 },
3306 );
3307 }
3308 attempts += 1;
3309 self.state.event(
3310 "fix",
3311 format!(
3312 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3313 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3314 ),
3315 );
3316 let mut retry = job.clone();
3317 retry.seat = seat.clone();
3318 retry.prompt = prompt::resume_incomplete(&last_err);
3319 retry.timeout = retry_budget(job.timeout, true);
3320 retry.stem = format!("{}-continue{attempts}", job.stem);
3321 let cache = self.state.config.cache_dir();
3322 let ctx = WaveCtx {
3323 carry_seats: false,
3324 pauses: Some((&self.pause, &self.interrupt)),
3325 run: run_id,
3326 node: "fix",
3327 prompts,
3328 cache: cache.as_deref(),
3329 round: Some(round),
3330 };
3331 let (resumed_seat, resumed_out) = run_one(
3332 retry,
3333 Arc::clone(&self.sem),
3334 &ctx,
3335 &mut self.state,
3336 attempts,
3337 )
3338 .await;
3339 seat = resumed_seat;
3340 match resumed_out {
3341 AgentOutcome::Ok(o) => {
3342 cumulative_wait_ms += o.duration_ms;
3343 match verdict::extract_json::<FixReport>(&o.text) {
3344 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3345 self.state.event(
3346 "fix",
3347 format!(
3348 "round {round}: fixer's adoption report recovered after \
3349 {attempts} continuation(s)"
3350 ),
3351 );
3352 return (
3353 seat,
3354 Some(report),
3355 None,
3356 ContinuationRecord {
3357 attempts,
3358 cumulative_wait_ms,
3359 outcome: ContinuationOutcome::Resumed,
3360 },
3361 );
3362 }
3363 Ok(_) => {
3371 last_err = "the reply parsed, but it reported a command whose own CLI \
3372 never confirmed an exit status"
3373 .to_owned();
3374 }
3375 Err(e) => last_err = e.to_string(),
3376 }
3377 }
3378 AgentOutcome::Quota(o) => {
3379 cumulative_wait_ms += o.duration_ms;
3380 self.state.quota.push(QuotaLoss {
3381 seat: seat.key.clone(),
3382 node: "fix".to_owned(),
3383 at: Timestamp::now(),
3384 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3385 });
3386 self.state.event(
3387 "fix",
3388 format!(
3389 "round {round}: continuation rate limited (quota); not retrying now"
3390 ),
3391 );
3392 return (
3393 seat,
3394 None,
3395 Some("rate limited (quota) while recovering the fix report".to_owned()),
3396 ContinuationRecord {
3397 attempts,
3398 cumulative_wait_ms,
3399 outcome: ContinuationOutcome::QuotaLost,
3400 },
3401 );
3402 }
3403 AgentOutcome::Dropped(o) => {
3404 cumulative_wait_ms += o.duration_ms;
3405 let why = o
3406 .dropped
3407 .as_ref()
3408 .map(|d| d.why.as_str())
3409 .unwrap_or("the CLI ended the stream without delivering its answer");
3410 last_err = format!("the CLI dropped the stream ({why})");
3411 }
3412 AgentOutcome::Failed(e) => last_err = e,
3413 AgentOutcome::ResourceBlocked(e) => {
3416 return (
3417 seat,
3418 None,
3419 Some(format!("the continuation was not started: {e}")),
3420 ContinuationRecord {
3421 attempts: attempts.saturating_sub(1),
3422 cumulative_wait_ms,
3423 outcome: ContinuationOutcome::NoSession,
3424 },
3425 );
3426 }
3427 }
3428 }
3429 }
3430
3431 fn after_implement(&mut self) -> Result<()> {
3432 if !self.state.candidate_preservation.is_empty() {
3433 bail!("candidate preservation incomplete; judging is unsafe");
3434 }
3435 if self.state.leaks.is_empty() {
3437 let cfg = self.state.config.blind.clone();
3438 let mut leaks = Vec::new();
3439 for c in &self.state.candidates {
3440 let Some(patch) =
3441 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3442 else {
3443 continue;
3444 };
3445 leaks.extend(blind::scan(
3446 &format!("candidate {} patch", c.label),
3447 &patch,
3448 &cfg.vendor_tokens,
3449 ));
3450 }
3451 if !leaks.is_empty() {
3452 let summary = leaks
3453 .iter()
3454 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3455 .collect::<Vec<_>>()
3456 .join(", ");
3457 match cfg.on_leak {
3458 LeakPolicy::Fail => {
3459 self.state.status = RunStatus::Failed;
3460 self.state
3461 .event("blind", format!("vendor text in a patch: {summary}"));
3462 self.state.leaks = leaks;
3463 self.state.save()?;
3464 self.settle_questions();
3465 bail!(
3466 "blind.on_leak = \"fail\" and vendor text reached a \
3467 judged patch: {summary}"
3468 );
3469 }
3470 LeakPolicy::Redact => self.state.event(
3471 "blind",
3472 format!("redacting vendor text for judging: {summary}"),
3473 ),
3474 LeakPolicy::Warn => self.state.event(
3475 "blind",
3476 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3477 ),
3478 }
3479 self.state.leaks = leaks;
3480 }
3481 }
3482
3483 if self.state.viable().is_empty() {
3484 if self.state.all_candidates_verified_noop() {
3485 self.state.status = RunStatus::VerifiedNoop;
3496 self.state.save()?;
3497 self.settle_questions();
3498 return Ok(());
3499 }
3500 self.state.status = RunStatus::Failed;
3501 self.state.save()?;
3502 self.settle_questions();
3503 bail!("no candidate produced a change; nothing to judge");
3504 }
3505 self.state.status = RunStatus::Judging;
3506 self.state.save()?;
3507 Ok(())
3508 }
3509
3510 async fn judge(&mut self) -> Result<()> {
3513 let run_id = self.state.id.clone();
3518 let prompts = self.state.config.prompts.clone();
3519 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3520 return Ok(());
3521 }
3522 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3523 if viable.len() == 1 {
3524 self.state.judge_skipped = true;
3531 self.state.event(
3532 "judge",
3533 format!(
3534 "only candidate {} produced a change; judging skipped",
3535 viable[0].label
3536 ),
3537 );
3538 self.state.save()?;
3539 return Ok(());
3540 }
3541 self.state.status = RunStatus::Judging;
3542
3543 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3544 let language = self.state.config.graph.language.clone();
3545 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3546 let sessions = self.state.config.graph.sessions;
3547 let artifacts = agent::artifacts_dir(&self.state.dir());
3548 let root = self.state.worktree_root();
3549 let base_short = short(&self.state.base_commit);
3550
3551 let mut jobs = Vec::new();
3552 let mut orders = Vec::new();
3553 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3554 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3555 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3556 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3557 let seat_key = format!("judge-{}", j + 1);
3558 let seat = self.seat(&seat_key, &spec.id);
3559 jobs.push(SeatJob {
3560 prompt: prompt::judge(
3561 &self.state.instruction,
3562 &views,
3563 self.roles.judges.len(),
3564 &base_short,
3565 &language,
3566 ),
3567 spec,
3568 seat,
3569 cwd: root.join(format!("judge-{}", j + 1)),
3570 timeout,
3571 allow_write: false,
3572 sessions,
3573 artifacts: artifacts.clone(),
3574 stem: format!("judge-{}", j + 1),
3575 handover: None,
3576 });
3577 }
3578
3579 self.state.event(
3580 "judge",
3581 format!(
3582 "{} judges ranking {} candidates blind",
3583 jobs.len(),
3584 viable.len()
3585 ),
3586 );
3587 let labels_for_check = labels.clone();
3588 let mut quota_losses = Vec::new();
3589 let cache = self.state.config.cache_dir();
3590 let ctx = WaveCtx {
3591 carry_seats: false,
3592 pauses: Some((&self.pause, &self.interrupt)),
3593 run: &run_id,
3594 node: "judge",
3595 prompts: &prompts,
3596 cache: cache.as_deref(),
3597 round: None,
3598 };
3599 let results = ask_json_wave::<Ranking>(
3600 jobs,
3601 Arc::clone(&self.sem),
3602 self.state.config.graph.retries,
3603 &self.roles.judge_roster,
3604 &ctx,
3605 &mut quota_losses,
3606 &mut self.state,
3607 &move |r: &Ranking| r.validate(&labels_for_check),
3608 )
3609 .await;
3610 if self.state.resource_block.is_some() {
3611 return self.stop_for_resource();
3612 }
3613 self.state.quota.extend(quota_losses);
3614
3615 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3616 let agent_id = seat.agent.clone();
3617 self.state.seats.insert(seat.key.clone(), seat);
3618 let mut record = Judgement {
3619 judge: j + 1,
3620 seat: format!("judge-{}", j + 1),
3621 agent: agent_id,
3622 ranking: Vec::new(),
3623 reasons: BTreeMap::new(),
3624 confidence: None,
3625 strengths: BTreeMap::new(),
3626 order: orders[j].clone(),
3627 failed: None,
3628 duration_ms: 0,
3629 };
3630 match res {
3631 Ok((ranking, out)) => {
3632 record.ranking = ranking.normalized();
3633 record.reasons = ranking.reasons;
3634 record.strengths = ranking.strengths;
3635 record.confidence = ranking.confidence;
3636 record.duration_ms = out.duration_ms;
3637 self.state.event(
3638 "judge",
3639 format!(
3640 "judge {} ranked {}",
3641 j + 1,
3642 record.ranking.iter().collect::<String>()
3643 ),
3644 );
3645 }
3646 Err(e) => {
3647 record.failed = Some(e.to_string());
3648 self.state
3649 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3650 }
3651 }
3652 self.state.judgements.push(record);
3653 self.state.save()?;
3654 }
3655 Ok(())
3656 }
3657
3658 async fn deliberate(&mut self) -> Result<()> {
3661 if self.hold_base_sync().await? {
3662 return Ok(());
3663 }
3664 let run_id = self.state.id.clone();
3669 let prompts = self.state.config.prompts.clone();
3670 if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3671 return Ok(());
3672 }
3673 let tops: Vec<char> = self
3674 .state
3675 .judgements
3676 .iter()
3677 .filter_map(|j| j.ranking.first().copied())
3678 .collect();
3679 let rounds = self.state.config.graph.deliberate_rounds;
3680 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3681 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3682 self.state.event(
3683 "deliberate",
3684 format!("judges agreed on {} outright; no deliberation", tops[0]),
3685 );
3686 }
3687 self.state.status = RunStatus::Voting;
3688 self.state.save()?;
3689 return Ok(());
3690 }
3691
3692 self.state.status = RunStatus::Deliberating;
3693 self.state.event(
3694 "deliberate",
3695 format!(
3696 "split: first choices were {} — opening {rounds} round(s)",
3697 tops.iter().collect::<String>()
3698 ),
3699 );
3700
3701 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3702 let language = self.state.config.graph.language.clone();
3703 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3704 let sessions = self.state.config.graph.sessions;
3705 let artifacts = agent::artifacts_dir(&self.state.dir());
3706 let root = self.state.worktree_root();
3707 let base_short = short(&self.state.base_commit);
3708
3709 for round in 1..=rounds {
3713 let mut turns: Vec<DeliberationTurn> = Vec::new();
3714 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3715 if self.state.judgements[j].failed.is_some() {
3716 continue;
3717 }
3718 let seat_key = format!("judge-{}", j + 1);
3719 let spec = self.occupant(&seat_key, spec);
3720 let mut seat = self.seat(&seat_key, &spec.id);
3721 let transcript = self.transcript(&turns, j);
3722 let build = |context: Option<&str>| {
3723 prompt::deliberate(
3724 &self.state.instruction,
3725 context,
3726 &transcript,
3727 round,
3728 rounds,
3729 &language,
3730 )
3731 };
3732 let block = self.candidate_block(&viable, &base_short);
3733 let full = build(Some(&block));
3734 let text = if has_context(&spec, &seat, sessions) {
3735 build(None)
3736 } else {
3737 full.clone()
3738 };
3739 let job = SeatJob {
3740 spec,
3741 seat: seat.clone(),
3742 prompt: text,
3743 cwd: root.join(format!("judge-{}", j + 1)),
3744 timeout,
3745 allow_write: false,
3746 sessions,
3747 artifacts: artifacts.clone(),
3748 stem: format!("delib-{round}-judge-{}", j + 1),
3749 handover: Some(full),
3750 };
3751 let cache = self.state.config.cache_dir();
3752 let ctx = WaveCtx {
3753 carry_seats: false,
3754 pauses: Some((&self.pause, &self.interrupt)),
3755 run: &run_id,
3756 node: "deliberate",
3757 prompts: &prompts,
3758 cache: cache.as_deref(),
3759 round: None,
3760 };
3761 let mut losses = Vec::new();
3765 let mut results = ask_wave_with::<String>(
3766 vec![job],
3767 Arc::clone(&self.sem),
3768 0,
3769 &self.roles.judge_roster,
3770 &ctx,
3771 &mut losses,
3772 &mut self.state,
3773 &|text: &str| {
3774 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3775 },
3776 )
3777 .await;
3778 self.state.quota.extend(losses);
3779 let (updated, res, _) = results.pop().expect("one job in, one result out");
3780 seat = updated;
3781 let agent_id = seat.agent.clone();
3782 self.state.seats.insert(seat.key.clone(), seat);
3783 let body = match res {
3784 Ok((body, _)) => body,
3785 Err(e) => {
3788 self.state
3789 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3790 continue;
3791 }
3792 };
3793 let tentative = verdict::extract_json::<Position>(&body)
3794 .ok()
3795 .and_then(|p| p.tentative)
3796 .and_then(|s| s.trim().chars().next())
3797 .map(|c| c.to_ascii_uppercase());
3798 self.state.event(
3799 "deliberate",
3800 format!(
3801 "round {round}: judge {} now favours {}",
3802 j + 1,
3803 tentative.map_or("—".to_owned(), |c| c.to_string())
3804 ),
3805 );
3806 turns.push(DeliberationTurn {
3807 judge: j + 1,
3808 agent: agent_id,
3809 body: blind::sanitize_prose(&body, &self.state.config.blind),
3810 tentative,
3811 });
3812 }
3813 self.state
3814 .deliberation
3815 .push(DeliberationRound { round, turns });
3816 self.state.save()?;
3817 }
3818
3819 self.state.status = RunStatus::Voting;
3820 self.state.save()?;
3821 Ok(())
3822 }
3823
3824 async fn vote(&mut self) -> Result<()> {
3827 if self.hold_base_sync().await? {
3828 return Ok(());
3829 }
3830 let run_id = self.state.id.clone();
3835 let prompts = self.state.config.prompts.clone();
3836 if !self.state.votes.is_empty() || self.state.tally.is_some() {
3837 return Ok(());
3838 }
3839 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3840 if viable.len() == 1 {
3841 return Ok(());
3842 }
3843 self.state.status = RunStatus::Voting;
3844
3845 let language = self.state.config.graph.language.clone();
3846 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3847 let sessions = self.state.config.graph.sessions;
3848 let artifacts = agent::artifacts_dir(&self.state.dir());
3849 let root = self.state.worktree_root();
3850 let base_short = short(&self.state.base_commit);
3851 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3852
3853 let mut jobs = Vec::new();
3854 let mut seats_at = Vec::new();
3855 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3856 if self
3857 .state
3858 .judgements
3859 .get(j)
3860 .is_some_and(|r| r.failed.is_some())
3861 {
3862 continue;
3863 }
3864 let seat_key = format!("judge-{}", j + 1);
3865 let spec = self.occupant(&seat_key, spec);
3866 let seat = self.seat(&seat_key, &spec.id);
3867 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3868 let text = if has_context(&spec, &seat, sessions) {
3869 prompt::final_vote(&viable, &language)
3870 } else {
3871 full.clone()
3872 };
3873 jobs.push(SeatJob {
3874 spec,
3875 seat,
3876 prompt: text,
3877 cwd: root.join(format!("judge-{}", j + 1)),
3878 timeout,
3879 allow_write: false,
3880 sessions,
3881 artifacts: artifacts.clone(),
3882 stem: format!("vote-judge-{}", j + 1),
3883 handover: Some(full),
3884 });
3885 seats_at.push(j);
3886 }
3887
3888 self.state.event(
3889 "vote",
3890 format!(
3891 "collecting {} final votes one by one, privately",
3892 jobs.len()
3893 ),
3894 );
3895 let allowed = viable.clone();
3896 let mut quota_losses = Vec::new();
3897 let cache = self.state.config.cache_dir();
3898 let ctx = WaveCtx {
3899 carry_seats: false,
3900 pauses: Some((&self.pause, &self.interrupt)),
3901 run: &run_id,
3902 node: "vote",
3903 prompts: &prompts,
3904 cache: cache.as_deref(),
3905 round: None,
3906 };
3907 let results = ask_json_wave::<FinalVote>(
3908 jobs,
3909 Arc::clone(&self.sem),
3910 self.state.config.graph.retries,
3911 &self.roles.judge_roster,
3912 &ctx,
3913 &mut quota_losses,
3914 &mut self.state,
3915 &move |v: &FinalVote| match v.label() {
3916 Some(c) if allowed.contains(&c) => Ok(()),
3917 other => bail!("vote {other:?} is not one of {allowed:?}"),
3918 },
3919 )
3920 .await;
3921 if self.state.resource_block.is_some() {
3922 return self.stop_for_resource();
3923 }
3924 self.state.quota.extend(quota_losses);
3925
3926 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3927 let agent_id = seat.agent.clone();
3928 self.state.seats.insert(seat.key.clone(), seat);
3929 let initial = self
3930 .state
3931 .judgements
3932 .get(j)
3933 .and_then(|r| r.ranking.first().copied());
3934 let mut record = VoteRecord {
3935 judge: j + 1,
3936 agent: agent_id,
3937 vote: None,
3938 reason: String::new(),
3939 changed: false,
3940 };
3941 match res {
3942 Ok((v, _)) => {
3943 record.vote = v.label();
3944 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3945 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3946 self.state.event(
3947 "vote",
3948 format!(
3949 "judge {} voted {}{}",
3950 j + 1,
3951 record.vote.unwrap_or('?'),
3952 if record.changed { " (changed)" } else { "" }
3953 ),
3954 );
3955 }
3956 Err(e) => {
3957 self.state
3958 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3959 }
3960 }
3961 self.state.votes.push(record);
3962 self.state.save()?;
3963 }
3964 Ok(())
3965 }
3966
3967 fn tally(&mut self) -> Result<()> {
3970 if self.state.tally.is_some() {
3971 return Ok(());
3972 }
3973 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3974 let tops: Vec<char> = self
3975 .state
3976 .judgements
3977 .iter()
3978 .filter_map(|j| j.ranking.first().copied())
3979 .collect();
3980 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3981
3982 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3985 let mut cast: Vec<char> = Vec::new();
3986 for (i, j) in self.state.judgements.iter().enumerate() {
3987 let vote = self
3988 .state
3989 .votes
3990 .iter()
3991 .find(|v| v.judge == i + 1)
3992 .and_then(|v| v.vote)
3993 .or_else(|| j.ranking.first().copied());
3994 if let Some(v) = vote {
3995 *first_choice.entry(v).or_insert(0) += 1;
3996 cast.push(v);
3997 }
3998 }
3999
4000 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4001 for j in &self.state.judgements {
4002 let n = j.ranking.len();
4003 for (pos, label) in j.ranking.iter().enumerate() {
4004 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4005 }
4006 }
4007
4008 let best = first_choice.values().copied().max().unwrap_or(0);
4009 let mut leaders: Vec<char> = first_choice
4010 .iter()
4011 .filter(|(_, v)| **v == best)
4012 .map(|(k, _)| *k)
4013 .collect();
4014 let mut tie_break = None;
4015 if leaders.len() > 1 {
4016 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4017 let borda_leaders: Vec<char> = leaders
4018 .iter()
4019 .copied()
4020 .filter(|l| borda[l] == top_borda)
4021 .collect();
4022 tie_break = Some(if borda_leaders.len() == 1 {
4023 format!(
4024 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4025 leaders.len()
4026 )
4027 } else {
4028 format!(
4029 "{} way tie on both first-choice votes and Borda points, broken by label order",
4030 leaders.len()
4031 )
4032 });
4033 leaders = borda_leaders;
4034 leaders.sort_unstable();
4035 }
4036 let winner = *leaders
4037 .first()
4038 .or(viable.first())
4039 .context("no candidate to declare a winner from")?;
4040
4041 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4042 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4043 let deliberated = !self.state.deliberation.is_empty();
4044
4045 let quota_seats: std::collections::BTreeSet<&str> =
4049 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4050 let mut present = 0usize;
4051 for (i, j) in self.state.judgements.iter().enumerate() {
4052 if quota_seats.contains(j.seat.as_str()) {
4053 continue;
4054 }
4055 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4056 let voted = self
4057 .state
4058 .votes
4059 .iter()
4060 .any(|v| v.judge == i + 1 && v.vote.is_some());
4061 if ranked || voted {
4062 present += 1;
4063 }
4064 }
4065 let needs_quorum = viable.len() > 1;
4071 let judges_total = if needs_quorum {
4072 self.roles.judges.len()
4073 } else {
4074 0
4075 };
4076 let quorum = if needs_quorum {
4077 judges_total / 2 + 1
4078 } else {
4079 0
4080 };
4081 let met_quorum = !needs_quorum || present >= quorum;
4082 let uncontested = (!needs_quorum).then(|| {
4083 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4084 });
4085
4086 self.state.event(
4087 "tally",
4088 match &uncontested {
4089 Some(reason) => format!("winner {winner} — {reason}"),
4090 None => format!(
4091 "winner {winner} — votes {} | initial {} | {} changed | \
4092 {present}/{judges_total} judges{}",
4093 first_choice
4094 .iter()
4095 .map(|(k, v)| format!("{k}:{v}"))
4096 .collect::<Vec<_>>()
4097 .join(" "),
4098 if unanimous_initial {
4099 "unanimous"
4100 } else {
4101 "split"
4102 },
4103 changed_votes,
4104 if met_quorum {
4105 String::new()
4106 } else {
4107 format!(" — below quorum ({quorum} required)")
4108 },
4109 ),
4110 },
4111 );
4112 if !met_quorum {
4113 self.state.event(
4114 "stall",
4115 format!(
4116 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4117 the run stops here, resumable"
4118 ),
4119 );
4120 }
4121 self.state.tally = Some(Tally {
4122 first_choice,
4123 borda,
4124 winner,
4125 rankings: tops.len(),
4126 unanimous_initial,
4127 deliberated,
4128 changed_votes,
4129 unanimous_final,
4130 tie_break,
4131 judges: judges_total,
4132 present,
4133 quorum,
4134 met_quorum,
4135 uncontested,
4136 });
4137 self.state.status = if met_quorum {
4138 RunStatus::Reviewing
4139 } else {
4140 RunStatus::Stalled
4141 };
4142 self.state.save()?;
4143 Ok(())
4144 }
4145
4146 #[allow(clippy::too_many_lines)]
4167 async fn recover_stall(&mut self) -> Result<bool> {
4168 let run_id = self.state.id.clone();
4173 let prompts = self.state.config.prompts.clone();
4174 let quota_seats: BTreeSet<&str> =
4179 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4180 let absent: Vec<String> = self
4181 .state
4182 .judgements
4183 .iter()
4184 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4185 .map(|j| j.seat.clone())
4186 .collect();
4187 if absent.is_empty() {
4188 return Ok(false);
4189 }
4190 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4191 if viable.len() <= 1 {
4192 return Ok(false);
4193 }
4194 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4195 let language = self.state.config.graph.language.clone();
4196 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4197 let sessions = self.state.config.graph.sessions;
4198 let artifacts = agent::artifacts_dir(&self.state.dir());
4199 let root = self.state.worktree_root();
4200 let base_short = short(&self.state.base_commit);
4201 let candidates: Vec<Candidate> = viable.clone();
4202
4203 let mut positions: Vec<usize> = absent
4205 .iter()
4206 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4207 .collect();
4208 if positions.is_empty() {
4209 return Ok(false);
4210 }
4211 positions.sort_unstable();
4212 positions.dedup();
4213
4214 let mut judge_jobs = Vec::new();
4216 for &j in &positions {
4217 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4218 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4219 let seat_key = format!("judge-{}", j + 1);
4220 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4221 let seat = self.seat(&seat_key, &spec.id);
4222 judge_jobs.push(SeatJob {
4223 spec,
4224 seat,
4225 prompt: prompt::judge(
4226 &self.state.instruction,
4227 &views,
4228 self.roles.judges.len(),
4229 &base_short,
4230 &language,
4231 ),
4232 cwd: root.join(seat_key),
4233 timeout,
4234 allow_write: false,
4235 sessions,
4236 artifacts: artifacts.clone(),
4237 stem: format!("judge-{}-recover", j + 1),
4238 handover: None,
4239 });
4240 }
4241
4242 let labels_for_check = labels.clone();
4243 let mut judge_losses = Vec::new();
4244 let retries = self.state.config.graph.retries;
4245 let cache = self.state.config.cache_dir();
4246 let ctx = WaveCtx {
4247 carry_seats: false,
4248 pauses: Some((&self.pause, &self.interrupt)),
4249 run: &run_id,
4250 node: "judge",
4251 prompts: &prompts,
4252 cache: cache.as_deref(),
4253 round: None,
4254 };
4255 let results = ask_json_wave::<Ranking>(
4256 judge_jobs,
4257 Arc::clone(&self.sem),
4258 retries,
4259 &self.roles.judge_roster,
4260 &ctx,
4261 &mut judge_losses,
4262 &mut self.state,
4263 &move |r: &Ranking| r.validate(&labels_for_check),
4264 )
4265 .await;
4266 if self.state.resource_block.is_some() {
4267 self.stop_for_resource()?;
4268 return Ok(false);
4269 }
4270
4271 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4273 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4274 let agent_id = seat.agent.clone();
4275 self.state.seats.insert(seat.key.clone(), seat);
4276 let record = &mut self.state.judgements[j];
4277 match res {
4278 Ok((ranking, out)) => {
4279 record.agent = agent_id;
4280 record.ranking = ranking.normalized();
4281 record.reasons = ranking.reasons;
4282 record.strengths = ranking.strengths;
4283 record.confidence = ranking.confidence;
4284 record.failed = None;
4285 record.duration_ms = out.duration_ms;
4286 recovered.insert(j);
4287 self.state.event(
4288 "recover",
4289 format!("judge {} ranked again after the limit", j + 1),
4290 );
4291 }
4292 Err(e) => {
4293 self.state
4294 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4295 }
4296 }
4297 }
4298
4299 let mut vote_jobs = Vec::new();
4301 let mut vote_pos: Vec<usize> = Vec::new();
4302 for &j in &recovered {
4303 let seat_key = format!("judge-{}", j + 1);
4304 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4305 let seat = self.seat(&seat_key, &spec.id);
4306 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4307 let text = if has_context(&spec, &seat, sessions) {
4308 prompt::final_vote(&labels, &language)
4309 } else {
4310 full.clone()
4311 };
4312 vote_jobs.push(SeatJob {
4313 spec,
4314 seat,
4315 prompt: text,
4316 cwd: root.join(seat_key),
4317 timeout,
4318 allow_write: false,
4319 sessions,
4320 artifacts: artifacts.clone(),
4321 stem: format!("vote-judge-{}-recover", j + 1),
4322 handover: Some(full),
4323 });
4324 vote_pos.push(j);
4325 }
4326 let allowed = labels.clone();
4327 let mut vote_losses = Vec::new();
4328 let vote_retries = self.state.config.graph.retries;
4329 let vote_cache = self.state.config.cache_dir();
4330 let ctx = WaveCtx {
4331 carry_seats: false,
4332 pauses: Some((&self.pause, &self.interrupt)),
4333 run: &run_id,
4334 node: "vote",
4335 prompts: &prompts,
4336 cache: vote_cache.as_deref(),
4337 round: None,
4338 };
4339 let votes = ask_json_wave::<FinalVote>(
4340 vote_jobs,
4341 Arc::clone(&self.sem),
4342 vote_retries,
4343 &self.roles.judge_roster,
4344 &ctx,
4345 &mut vote_losses,
4346 &mut self.state,
4347 &move |v: &FinalVote| match v.label() {
4348 Some(c) if allowed.contains(&c) => Ok(()),
4349 other => bail!("vote {other:?} is not one of {allowed:?}"),
4350 },
4351 )
4352 .await;
4353 if self.state.resource_block.is_some() {
4354 self.stop_for_resource()?;
4355 return Ok(false);
4356 }
4357 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4358 let agent_id = seat.agent.clone();
4359 self.state.seats.insert(seat.key.clone(), seat);
4360 match res {
4361 Ok((v, _)) => {
4362 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4363 rec.vote = v.label();
4364 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4365 } else {
4366 self.state.votes.push(VoteRecord {
4367 judge: j + 1,
4368 agent: agent_id,
4369 vote: v.label(),
4370 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4371 changed: false,
4372 });
4373 }
4374 self.state.event(
4375 "recover",
4376 format!("judge {} voted again after the limit", j + 1),
4377 );
4378 }
4379 Err(e) => {
4380 self.state
4381 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4382 }
4383 }
4384 }
4385
4386 let recovered_keys: BTreeSet<String> = recovered
4390 .iter()
4391 .map(|&j| format!("judge-{}", j + 1))
4392 .collect();
4393 self.state
4394 .quota
4395 .retain(|q| !recovered_keys.contains(&q.seat));
4396 for loss in judge_losses.into_iter().chain(vote_losses) {
4400 if recovered_keys.contains(&loss.seat) {
4401 continue;
4402 }
4403 self.state.quota.retain(|q| q.seat != loss.seat);
4404 self.state.quota.push(loss);
4405 }
4406
4407 self.state.tally = None;
4409 self.tally()?;
4410 Ok(self
4411 .state
4412 .tally
4413 .as_ref()
4414 .map(|t| t.met_quorum)
4415 .unwrap_or(false))
4416 }
4417
4418 async fn graft(&mut self) -> Result<()> {
4434 let Err(e) = self.graft_inner().await else {
4435 return Ok(());
4436 };
4437 if e.is::<crate::repo_lock::ResourceBlocked>() {
4438 return Err(e);
4439 }
4440 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4441 self.state.status = RunStatus::Blocked;
4442 self.state.event("graft", format!("blocked: {e:#}"));
4443 self.state.save()?;
4444 return Err(e);
4445 }
4446 if let Some(rec) = self.state.graft.clone()
4451 && !rec.outcome.settled()
4452 {
4453 if let Some(winner) = self.state.winner().cloned() {
4454 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4455 if re.is::<crate::repo_lock::ResourceBlocked>() {
4456 return Err(re);
4457 }
4458 self.state.status = RunStatus::Blocked;
4459 self.state.event("graft", format!("blocked: {re:#}"));
4460 self.state.save()?;
4461 return Err(re);
4462 }
4463 }
4464 if let Some(g) = self.state.graft.as_mut() {
4465 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4466 }
4467 }
4468 self.state.event("graft", format!("skipped: {e:#}"));
4469 if let Err(e) = self.state.save() {
4470 tracing::warn!("save after graft: {e:#}");
4471 }
4472 Ok(())
4473 }
4474
4475 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4478 let _coordination = self.coordinate(wt).await?;
4479 crate::recovery::restore_repair(wt, pre)
4480 .await
4481 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4482 }
4483
4484 fn graft_skipped(&mut self, why: &str) {
4486 let message = format!("skipped: {why}");
4487 if !self
4488 .state
4489 .events
4490 .iter()
4491 .any(|e| e.node == "graft" && e.message == message)
4492 {
4493 self.state.event("graft", message);
4494 }
4495 }
4496
4497 async fn graft_inner(&mut self) -> Result<()> {
4498 if let Some(rec) = self.state.graft.clone()
4501 && !rec.outcome.settled()
4502 {
4503 if self.state.reviews.is_empty()
4507 && let Some(winner) = self.state.winner().cloned()
4508 {
4509 self.graft_restore(&winner.worktree, &rec.pre).await?;
4510 }
4511 if rec.outcome == GraftOutcome::WaitingForRestore {
4512 self.state.graft = None;
4513 self.state.save()?;
4514 } else {
4515 if let Some(g) = self.state.graft.as_mut() {
4516 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4517 }
4518 self.state.event(
4519 "graft",
4520 "rolled back: the previous round was interrupted; the winner is untouched",
4521 );
4522 self.state.save()?;
4523 return Ok(());
4524 }
4525 }
4526 let plan = match graft::plan(&self.state) {
4527 Ok(plan) => plan,
4528 Err(skip) => {
4529 if skip != graft::Skip::AlreadyDone {
4530 self.graft_skipped(skip.reason());
4531 self.state.save()?;
4532 }
4533 return Ok(());
4534 }
4535 };
4536 let Some(winner) = self.state.winner().cloned() else {
4537 return Ok(());
4538 };
4539 let wt = winner.worktree.clone();
4540 if !git::is_clean(&wt).await? {
4541 self.graft_skipped("the winner's worktree has uncommitted changes");
4542 self.state.save()?;
4543 return Ok(());
4544 }
4545 let pre = git::rev_parse(&wt, "HEAD").await?;
4546 self.state.graft = Some(GraftRecord {
4549 pre: pre.clone(),
4550 post: None,
4551 lines: None,
4552 limit: None,
4553 agent: None,
4554 outcome: GraftOutcome::Running,
4555 requested: graft::requested(&plan),
4556 adoption: None,
4557 adoption_error: None,
4558 });
4559 self.state.event(
4560 "graft",
4561 format!(
4562 "porting strengths of candidate(s) {} onto {}",
4563 plan.strengths
4564 .iter()
4565 .map(|(l, _, _)| l.to_string())
4566 .collect::<Vec<_>>()
4567 .join(", "),
4568 plan.winner
4569 ),
4570 );
4571 self.state.save()?;
4572
4573 let fix_prompt = prompt::graft(
4574 &self.state.instruction,
4575 plan.winner,
4576 &plan.strengths,
4577 &self.state.config.graph.language,
4578 );
4579 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4580 let sessions = self.state.config.graph.sessions;
4581 let artifacts = agent::artifacts_dir(&self.state.dir());
4582 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4583 let (_, seat, out) = self
4584 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4585 SeatJob {
4586 prompt: fix_prompt.clone(),
4587 spec,
4588 seat,
4589 cwd: wt.clone(),
4590 timeout,
4591 allow_write: true,
4592 sessions,
4593 artifacts: artifacts.clone(),
4594 stem: "graft".to_owned(),
4595 handover: None,
4596 }
4597 })
4598 .await;
4599 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4600 self.state.graft.as_mut().expect("graft record").outcome =
4603 GraftOutcome::WaitingForRestore;
4604 return self.stop_for_resource();
4605 }
4606 let mut why: Option<String> = None;
4607 match &out {
4608 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4609 why = Some("the fixer produced nothing usable".to_owned());
4610 }
4611 AgentOutcome::Ok(_) => {}
4612 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4613 AgentOutcome::Quota(o) => {
4614 self.state.quota.push(QuotaLoss {
4615 seat: seat.key.clone(),
4616 node: "graft".to_owned(),
4617 at: Timestamp::now(),
4618 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4619 });
4620 why = Some("rate limited (quota)".to_owned());
4621 }
4622 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4623 why = Some(format!(
4624 "the fixer failed: {}",
4625 e.lines()
4626 .next()
4627 .unwrap_or("")
4628 .chars()
4629 .take(160)
4630 .collect::<String>()
4631 ));
4632 }
4633 }
4634 let agent_id = seat.agent.clone();
4635 self.state.seats.insert(seat.key.clone(), seat);
4636
4637 let mut post = None;
4638 let mut lines = None;
4639 let _coordination = self.coordinate(&wt).await?;
4640 if why.is_none() {
4641 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4642 if let Ok(r) = &rescued {
4643 self.state.note_withheld("graft", &r.withheld);
4644 }
4645 let head = git::rev_parse(&wt, "HEAD").await?;
4646 let dirty = match &rescued {
4650 Ok(r) => git::status_porcelain(&wt)
4651 .await
4652 .map(|st| {
4653 st.lines().any(|l| {
4654 !r.withheld
4655 .iter()
4656 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4657 })
4658 })
4659 .unwrap_or(true),
4660 Err(_) => true,
4661 };
4662 if dirty {
4663 why = Some("the fixer left changes that could not be committed".to_owned());
4664 } else if head == pre {
4665 why = Some("the fixer changed nothing".to_owned());
4666 } else {
4667 lines = git::changed_lines(&wt, &pre, &head).await;
4668 post = Some(head);
4669 }
4670 }
4671 let (adoption, adoption_error) = match &out {
4672 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4673 _ => (None, None),
4674 };
4675 let requested = graft::requested(&plan);
4676 let outcome = match why {
4677 None => {
4678 self.state.event(
4679 "graft",
4680 format!(
4681 "applied: {}",
4682 lines.map_or_else(
4683 || "changed lines unavailable".to_owned(),
4684 |n| format!("{n} changed lines")
4685 )
4686 ),
4687 );
4688 GraftOutcome::Applied
4689 }
4690 Some(why) => {
4691 crate::recovery::restore_repair(&wt, &pre)
4694 .await
4695 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4696 self.state.event(
4697 "graft",
4698 format!("rolled back: {why}; the winner is untouched"),
4699 );
4700 GraftOutcome::RolledBack(why)
4701 }
4702 };
4703 self.state.graft = Some(GraftRecord {
4704 pre,
4705 post,
4706 lines,
4707 limit: None,
4708 agent: Some(agent_id),
4709 outcome,
4710 requested,
4711 adoption,
4712 adoption_error,
4713 });
4714 self.state.save()?;
4715 Ok(())
4716 }
4717
4718 async fn fold_losers(&mut self) -> Result<()> {
4721 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4722 return Ok(());
4723 };
4724 if !self.state.candidate_preservation.is_empty() {
4725 bail!("candidate preservation incomplete; refusing cleanup");
4726 }
4727 let repo = self.state.repo.clone();
4728 let _coordination = self.coordinate(&repo).await?;
4729 let mut folded = Vec::new();
4730 for i in 0..self.state.candidates.len() {
4731 let c = &self.state.candidates[i];
4732 if c.label == winner || c.folded {
4733 continue;
4734 }
4735 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4736 remove_preserved_worktree(&repo, &wt).await?;
4737 if git::branch_exists(&repo, &branch).await?
4738 && !git::branch_delete(&repo, &branch).await?
4739 {
4740 bail!("candidate branch removal refused");
4741 }
4742 self.state.candidates[i].folded = true;
4743 self.state.save()?;
4744 folded.push(label.to_string());
4745 }
4746 let root = self.state.worktree_root();
4748 for j in 1..=self.roles.judges.len() {
4749 let wt = root.join(format!("judge-{j}"));
4750 if wt.exists() {
4751 remove_preserved_worktree(&repo, &wt).await?;
4752 }
4753 }
4754 if self.state.config.graph.advise {
4757 for k in 1..=self.state.config.graph.advisors {
4758 let wt = root.join(format!("advisor-{k}"));
4759 if wt.exists() {
4760 remove_preserved_worktree(&repo, &wt).await?;
4761 }
4762 }
4763 }
4764 if !folded.is_empty() {
4765 self.state
4766 .event("fold", format!("folded candidates {}", folded.join(", ")));
4767 self.state.save()?;
4768 }
4769 Ok(())
4770 }
4771
4772 async fn hold_base_sync(&mut self) -> Result<bool> {
4775 self.hold_base_sync_as(None).await
4776 }
4777
4778 async fn hold_base_sync_kind(
4779 &mut self,
4780 kind: crate::base_sync_recovery::StopKind,
4781 ) -> Result<()> {
4782 self.hold_base_sync_as(Some(kind)).await?;
4783 Ok(())
4784 }
4785
4786 async fn hold_base_sync_as(
4787 &mut self,
4788 kind: Option<crate::base_sync_recovery::StopKind>,
4789 ) -> Result<bool> {
4790 if !self.state.base_sync_blocked() {
4791 self.state.base_sync_recovery.stop = None;
4792 return Ok(false);
4793 }
4794 let head = match self.state.winner() {
4795 Some(w) => git::rev_parse(&w.worktree, "HEAD")
4796 .await
4797 .unwrap_or_default(),
4798 None => String::new(),
4799 };
4800 self.state.record_base_sync_stop(&head);
4801 if let Some(kind) = kind {
4802 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4803 }
4804 self.state.save()?;
4805 Ok(true)
4806 }
4807
4808 async fn sync_to_base(&mut self) -> Result<()> {
4850 if self.state.status == RunStatus::AlreadyInBase {
4851 return Ok(());
4852 }
4853 self.hold_base_sync().await?;
4854 let conflicted = self
4855 .state
4856 .base_sync
4857 .as_ref()
4858 .is_some_and(|s| s.conflict.is_some());
4859 let Some(winner) = self.state.winner().cloned() else {
4860 return Ok(());
4861 };
4862
4863 let repo = self.state.repo.clone();
4864 let remote = self.state.config.merge.remote.clone();
4865 let base_branch = self.state.base_branch.clone();
4866 let tracking = format!("{remote}/{base_branch}");
4867
4868 let fetched_base =
4869 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4870 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4875 return Ok(());
4876 };
4877
4878 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4879 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4880 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4881
4882 if (behind > 0 || conflicted || head == tip)
4891 && self
4892 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4893 .await?
4894 {
4895 return Ok(());
4896 }
4897 if conflicted && self.state.base_recovery_pending() {
4898 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4899 let owner =
4900 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4901 if !fetched_base
4902 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4903 || !git::is_clean(&winner.worktree).await?
4904 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4905 {
4906 return Ok(());
4907 }
4908 }
4909 let recovery = conflicted && self.state.consume_base_recovery(&head);
4910 if conflicted {
4911 self.state.save()?;
4913 if !recovery {
4914 return Ok(());
4915 }
4916 }
4917
4918 if behind == 0 {
4919 if let Some(from) = self
4928 .state
4929 .rebase_fixes
4930 .iter()
4931 .rev()
4932 .find_map(|r| r.from.clone())
4933 && from != head
4934 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4935 .await
4936 .is_ok_and(|o| o.ok())
4937 {
4938 git::sync_to_head(&winner.worktree).await?;
4939 }
4940 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4946 if let Some(why) = &conflict {
4947 self.state.status = RunStatus::Blocked;
4948 self.state.event("land", why.clone());
4949 }
4950 self.state.base_sync = Some(BaseSync {
4951 tip,
4952 behind: 0,
4953 attempts: attempts + usize::from(recovery),
4954 conflict,
4955 already_in: None,
4956 });
4957 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4958 .await?;
4959 self.state.save()?;
4960 return Ok(());
4961 }
4962
4963 if attempts >= BASE_SYNC_ROUNDS && !recovery {
4964 let why = format!(
4965 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4966 synchronization(s); synchronizing again would only race it",
4967 winner.branch
4968 );
4969 self.state.status = RunStatus::Blocked;
4970 self.state.base_sync = Some(BaseSync {
4971 tip,
4972 behind,
4973 attempts,
4974 conflict: Some(why.clone()),
4975 already_in: None,
4976 });
4977 self.state.event("land", why);
4978 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4979 .await?;
4980 self.state.save()?;
4981 return Ok(());
4982 }
4983
4984 if self.state.foreign_checkout.is_some() {
4985 let wait = repository_wait(&self.state, None);
4988 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
4989 let merged = git::git_raw(
4990 &winner.worktree,
4991 &[
4992 "-c",
4993 "core.hooksPath=/dev/null",
4994 "merge",
4995 "--no-ff",
4996 "--no-autostash",
4997 "--no-edit",
4998 "-m",
4999 &format!("Merge {tracking} before verifying"),
5000 &tip,
5001 ],
5002 )
5003 .await?;
5004 let conflict = if merged.ok() {
5005 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5006 self.state.event(
5007 "land",
5008 format!("merged {tracking} into {} before verifying", winner.branch),
5009 );
5010 None
5011 } else {
5012 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5013 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5014 }
5015 let why = format!(
5016 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5017 winner.branch,
5018 merged.stderr.chars().take(600).collect::<String>()
5019 );
5020 self.state.status = RunStatus::Blocked;
5021 self.state.event("land", &why);
5022 Some(why)
5023 };
5024 let synced = conflict.is_none();
5025 self.state.base_sync = Some(BaseSync {
5026 tip,
5027 behind: if synced { 0 } else { behind },
5028 attempts: attempts + 1,
5029 conflict,
5030 already_in: None,
5031 });
5032 self.state.save()?;
5033 return Ok(());
5034 }
5035
5036 self.state.event(
5037 "land",
5038 format!(
5039 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5040 winner.branch
5041 ),
5042 );
5043 self.state.save()?;
5044
5045 let branch_tracking = format!("{remote}/{}", winner.branch);
5051 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5052 let remote_tip = if self.state.task_pr_branch().is_none()
5053 && matches!(&fetched_branch, Ok(o) if o.ok())
5054 {
5055 git::rev_parse(&repo, &branch_tracking).await.ok()
5056 } else {
5057 None
5058 };
5059 if let Some(theirs) = &remote_tip
5063 && !git::is_ancestor(&repo, theirs, &head).await
5064 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5065 .await
5066 .is_ok_and(|missing| missing.is_empty())
5067 {
5068 let why = format!(
5069 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5070 them",
5071 short(theirs),
5072 winner.branch
5073 );
5074 self.state.status = RunStatus::Blocked;
5075 self.state.base_sync = Some(BaseSync {
5076 tip,
5077 behind,
5078 attempts: attempts + usize::from(recovery),
5079 conflict: Some(why.clone()),
5080 already_in: None,
5081 });
5082 self.state.event("land", why);
5083 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5084 .await?;
5085 self.state.save()?;
5086 return Ok(());
5087 }
5088
5089 let scratch = self.state.dir().join("base-sync");
5090 let rebased =
5091 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5092 .await
5093 {
5094 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5095 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5096 Err(e) => Err(e),
5097 };
5098 let attempts = attempts + 1;
5099 let stop_kind = match &rebased {
5100 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5101 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5102 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5103 };
5104 match rebased {
5105 Ok(None) => {
5106 git::sync_to_head(&winner.worktree).await?;
5110 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5111 let mut conflict = None;
5112 if let Some(pinned) = &remote_tip {
5113 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5114 match pushed {
5115 Ok(o) if o.ok() => self.state.event(
5116 "land",
5117 format!("pushed rebased {} to {remote}", winner.branch),
5118 ),
5119 Ok(o) => {
5120 conflict = Some(format!(
5121 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5122 winner.branch,
5123 short(pinned),
5124 o.stderr.chars().take(600).collect::<String>()
5125 ));
5126 }
5127 Err(e) => {
5128 conflict = Some(format!(
5129 "rebased {} locally but could not push it: {e:#}",
5130 winner.branch
5131 ));
5132 }
5133 }
5134 }
5135 if let Some(why) = &conflict {
5136 self.state.status = RunStatus::Blocked;
5137 self.state.event("land", why.clone());
5138 }
5139 self.state.base_sync = Some(BaseSync {
5140 tip: tip.clone(),
5141 behind: 0,
5142 attempts,
5143 conflict,
5144 already_in: None,
5145 });
5146 self.state
5147 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5148 }
5149 Ok(Some(conflict)) => {
5150 let why = format!(
5151 "{} conflicts with {tracking} and did not rebase: {}",
5152 winner.branch,
5153 conflict.chars().take(600).collect::<String>()
5154 );
5155 self.state.status = RunStatus::Blocked;
5156 self.state.base_sync = Some(BaseSync {
5157 tip,
5158 behind,
5159 attempts,
5160 conflict: Some(why.clone()),
5161 already_in: None,
5162 });
5163 self.state.event("land", why);
5164 }
5165 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5169 self.state.save()?;
5170 return Ok(());
5171 }
5172 Err(e) => {
5173 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5174 self.state.status = RunStatus::Blocked;
5175 self.state.base_sync = Some(BaseSync {
5176 tip,
5177 behind,
5178 attempts,
5179 conflict: Some(why.clone()),
5180 already_in: None,
5181 });
5182 self.state.event("land", why);
5183 }
5184 }
5185 if !self.state.base_sync_blocked() {
5186 self.state.base_sync_recovery.stop = None;
5187 }
5188 self.hold_base_sync_kind(stop_kind).await?;
5189 self.state.save()?;
5190 Ok(())
5191 }
5192
5193 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5199 if self.state.task_pr_branch().is_some() {
5202 return None;
5203 }
5204 let repo = self.state.repo.clone();
5205 let remote = self.state.config.merge.remote.clone();
5206 let fetched = git::fetch(&repo, &remote, branch).await;
5207 if !matches!(&fetched, Ok(o) if o.ok()) {
5208 return None;
5209 }
5210 let branch_tracking = format!("{remote}/{branch}");
5211 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5212 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5213 return None;
5214 }
5215 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5216 .await
5217 .is_ok_and(|missing| missing.is_empty())
5218 {
5219 return Some(format!(
5220 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5221 them",
5222 short(&theirs)
5223 ));
5224 }
5225 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5226 Ok(o) if o.ok() => {
5227 self.state
5228 .event("land", format!("pushed rebased {branch} to {remote}"));
5229 None
5230 }
5231 Ok(o) => Some(format!(
5232 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5233 someone may have pushed): {}",
5234 short(&theirs),
5235 o.stderr.chars().take(600).collect::<String>()
5236 )),
5237 Err(e) => Some(format!(
5238 "{branch} is rebased locally but could not be pushed: {e:#}"
5239 )),
5240 }
5241 }
5242
5243 async fn settle_already_in(
5260 &mut self,
5261 branch: &str,
5262 tip: &str,
5263 head: &str,
5264 attempts: usize,
5265 behind: usize,
5266 ) -> Result<bool> {
5267 let repo = self.state.repo.clone();
5268 let remote = self.state.config.merge.remote.clone();
5269 let start = self.state.base_commit.clone();
5270 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5271 Ok(Some(e)) => e,
5272 Ok(None) => return Ok(false),
5273 Err(e) => {
5274 tracing::warn!("already-in-base check for {branch}: {e:#}");
5275 return Ok(false);
5276 }
5277 };
5278 let mut verified = vec![head.to_owned()];
5279 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5280 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5281 if matches!(&fetched, Ok(o) if o.ok())
5282 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5283 && theirs != head
5284 {
5285 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5286 Ok(Some(_)) => verified.push(theirs),
5287 _ => return Ok(false),
5288 }
5289 }
5290 let base_branch = self.state.base_branch.clone();
5291 let message = format!(
5292 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5293 land",
5294 evidence.names(),
5295 evidence.proof.as_str()
5296 );
5297 let closed =
5298 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5299 .await;
5300 match closed {
5301 Ok(Ok(url)) => self
5302 .state
5303 .event("land", format!("closed {url}: superseded on {base_branch}")),
5304 Ok(Err(why)) => self
5305 .state
5306 .event("land", format!("did not close a pull request: {why}")),
5307 Err(e) => {
5308 let why = format!(
5309 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5310 not be closed ({e:#}); resume to retry"
5311 );
5312 self.state.status = RunStatus::Blocked;
5313 self.state.base_sync = Some(BaseSync {
5314 tip: tip.to_owned(),
5315 behind,
5316 attempts,
5317 conflict: Some(why.clone()),
5318 already_in: None,
5319 });
5320 self.state.event("land", why);
5321 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5322 .await?;
5323 self.state.save()?;
5324 return Ok(true);
5325 }
5326 }
5327 self.state.status = RunStatus::AlreadyInBase;
5328 self.state.base_sync_recovery.stop = None;
5329 self.state.base_sync = Some(BaseSync {
5330 tip: tip.to_owned(),
5331 behind,
5332 attempts,
5333 conflict: None,
5334 already_in: Some(evidence),
5335 });
5336 self.state.event("land", message);
5337 self.state.save()?;
5338 self.settle_questions();
5339 Ok(true)
5340 }
5341
5342 fn landing_base(&self) -> String {
5352 self.state
5353 .base_sync
5354 .as_ref()
5355 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5356 }
5357
5358 pub async fn fix_selected(
5391 &mut self,
5392 ids: &[String],
5393 reason: &str,
5394 allow_stale: bool,
5395 ) -> Result<()> {
5396 let reason = reason.trim();
5397 if reason.is_empty() {
5398 bail!("a fix request needs a reason — that is the operator's own record of why");
5399 }
5400 if ids.is_empty() {
5401 bail!("no finding id given");
5402 }
5403 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5404 bail!(
5405 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5406 has already concluded — can be given a targeted fix. A run still \
5407 in progress should simply be resumed; a `merged` run's branch has \
5408 already landed, so its answer is a fresh `magi review <branch>`, \
5409 not reopening this run's own record",
5410 self.state.id,
5411 self.state.status.as_str()
5412 );
5413 }
5414 let Some(winner) = self.state.winner().cloned() else {
5415 bail!("run {} has no winning candidate to fix", self.state.id);
5416 };
5417 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5418 bail!(
5419 "branch `{}` no longer exists; this run cannot be extended",
5420 winner.branch
5421 );
5422 }
5423 let home = crate::run::home();
5424 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5425 bail!(
5426 "run {} is currently being worked on by another magi process",
5427 self.state.id
5428 );
5429 }
5430 let _claim = FixClaim::acquire(&self.state.dir())?;
5436
5437 let mut seen = BTreeSet::new();
5441 let mut findings = Vec::new();
5442 let mut missing = Vec::new();
5443 for id in ids {
5444 if !seen.insert(id.clone()) {
5445 continue;
5446 }
5447 match self.state.finding(id) {
5448 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5449 id: f.id.clone(),
5450 severity: f.severity,
5451 reviewer_vote: rec.vote,
5452 round: round.round,
5453 round_head: round.head.clone(),
5454 reviewer: rec.reviewer,
5455 agent: rec.agent.clone(),
5456 file: f.file.clone(),
5457 line: f.line,
5458 title: f.title.clone(),
5459 detail: f.detail.clone(),
5460 outcome: OperatorFixOutcome::Pending,
5461 }),
5462 None => missing.push(id.clone()),
5463 }
5464 }
5465 if !missing.is_empty() {
5466 bail!(
5467 "unknown finding id(s): {}; nothing was changed",
5468 missing.join(", ")
5469 );
5470 }
5471
5472 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5473 let stale_details: Vec<(String, String)> = findings
5474 .iter()
5475 .filter(|f| f.round_head != head_at_request)
5476 .map(|f| (f.id.clone(), f.round_head.clone()))
5477 .collect();
5478 let stale = !stale_details.is_empty();
5479 if stale && !allow_stale {
5480 bail!(
5481 "the branch has moved since some finding(s) were raised — {} — now \
5482 at {}; pass --allow-stale to fix anyway, or re-run review first",
5483 stale_details
5484 .iter()
5485 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5486 .collect::<Vec<_>>()
5487 .join(", "),
5488 short(&head_at_request)
5489 );
5490 }
5491
5492 let request = OperatorFixRequest {
5493 requested_at: Timestamp::now(),
5494 reason: reason.to_owned(),
5495 findings,
5496 head_at_request: head_at_request.clone(),
5497 allow_stale,
5498 stale,
5499 fix: None,
5500 result_head: None,
5501 follow_up_review_run: None,
5502 };
5503 self.state.event(
5504 "fix",
5505 format!(
5506 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5507 request.findings.len(),
5508 request
5509 .findings
5510 .iter()
5511 .map(|f| f.id.as_str())
5512 .collect::<Vec<_>>()
5513 .join(", "),
5514 ),
5515 );
5516 self.state.operator_fixes.push(request);
5523 self.state.save()?;
5524 let request_index = self.state.operator_fixes.len() - 1;
5525
5526 let repo = self.state.repo.clone();
5535 let coordination = self.coordinate(&repo).await?;
5536 if winner.worktree.exists() {
5537 let dirty = git::git(
5540 &winner.worktree,
5541 &["status", "--porcelain", "--untracked-files=all"],
5542 )
5543 .await?;
5544 let only_withheld = dirty.lines().all(|l| {
5545 l.strip_prefix("?? ")
5546 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5547 });
5548 if !only_withheld {
5549 bail!(
5550 "`{}` has uncommitted changes; refusing to touch it — commit or \
5551 discard them first",
5552 winner.worktree.display()
5553 );
5554 }
5555 git::worktree_remove(&self.state.repo, &winner.worktree)
5556 .await
5557 .ok();
5558 }
5559 let fix_worktree = self.state.worktree_root().join("operator-fix");
5560 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5561 git::git(
5562 &self.state.repo,
5563 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5564 )
5565 .await
5566 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5567 if !git::is_clean(&fix_worktree).await? {
5568 git::worktree_remove(&self.state.repo, &fix_worktree)
5569 .await
5570 .ok();
5571 bail!(
5572 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5573 winner.branch
5574 );
5575 }
5576 let repo_for_setup = self.state.repo.clone();
5577 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5578 Ok(()) => {
5579 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5580 }
5581 Err(e) => Err(e),
5582 };
5583 if let Err(e) = setup {
5584 git::worktree_remove(&self.state.repo, &fix_worktree)
5585 .await
5586 .ok();
5587 return Err(e);
5588 }
5589
5590 let run_id = self.state.id.clone();
5591 let prompts = self.state.config.prompts.clone();
5592 let language = self.state.config.graph.language.clone();
5593 let sessions = self.state.config.graph.sessions;
5594 let artifacts = agent::artifacts_dir(&self.state.dir());
5595 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5596 .findings
5597 .iter()
5598 .map(|f| Finding {
5599 id: f.id.clone(),
5600 severity: f.severity,
5601 file: f.file.clone(),
5602 line: f.line,
5603 title: f.title.clone(),
5604 detail: f.detail.clone(),
5605 })
5606 .collect();
5607 let fix_prompt = prompt::operator_fix_in(
5608 &self.state.instruction,
5609 &finding_list,
5610 reason,
5611 &stale_details,
5612 &head_at_request,
5613 &language,
5614 &self.state.config.graph.github_language,
5615 );
5616 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5617 drop(coordination);
5618 let (job, seat, out) = self
5619 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5620 prompt: fix_prompt.clone(),
5621 spec,
5622 seat,
5623 cwd: fix_worktree.clone(),
5624 timeout,
5625 allow_write: true,
5626 sessions,
5627 artifacts: artifacts.clone(),
5628 stem: "operator-fix".to_owned(),
5629 handover: None,
5630 })
5631 .await;
5632 let agent_id = seat.agent.clone();
5633
5634 let mut fix = FixRecord {
5635 agent: agent_id,
5636 addressed: Vec::new(),
5637 rejected: Vec::new(),
5638 notes: String::new(),
5639 committed: false,
5640 failed: None,
5641 duration_ms: 0,
5642 continuation: None,
5643 };
5644 let mut final_seat = seat.clone();
5645 match out {
5646 AgentOutcome::Ok(o) => {
5647 fix.duration_ms = o.duration_ms;
5648 let parsed = verdict::extract_json::<FixReport>(&o.text);
5649 let incomplete_reason = match &parsed {
5650 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5651 "the reply parsed, but it reported a command whose own CLI \
5652 never confirmed an exit status"
5653 .to_owned(),
5654 ),
5655 Ok(_) => None,
5656 Err(e) => Some(e.to_string()),
5657 };
5658 match incomplete_reason {
5659 None => {
5660 let report = parsed.expect("checked Ok above");
5661 fix.addressed = report.addressed;
5662 fix.rejected = report.rejected;
5663 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5664 }
5665 Some(reason) => {
5666 let (resumed_seat, resolved, failure, cont) = self
5667 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5668 .await;
5669 fix.duration_ms += cont.cumulative_wait_ms;
5670 fix.continuation = Some(cont);
5671 final_seat = resumed_seat;
5672 match resolved {
5673 Some(report) => {
5674 fix.addressed = report.addressed;
5675 fix.rejected = report.rejected;
5676 fix.notes =
5677 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5678 }
5679 None => fix.failed = failure,
5680 }
5681 }
5682 }
5683 }
5684 AgentOutcome::Dropped(o) => {
5685 fix.duration_ms = o.duration_ms;
5686 let why = o
5687 .dropped
5688 .as_ref()
5689 .map(|d| d.why.as_str())
5690 .unwrap_or("the CLI ended the stream without delivering its answer");
5691 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5692 }
5693 AgentOutcome::Quota(o) => {
5694 self.state.quota.push(QuotaLoss {
5695 seat: final_seat.key.clone(),
5696 node: "fix".to_owned(),
5697 at: Timestamp::now(),
5698 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5699 });
5700 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5701 }
5702 AgentOutcome::Failed(e) => fix.failed = Some(e),
5703 AgentOutcome::ResourceBlocked(e) => {
5704 fix.failed = Some(format!("the fixer was not started: {e}"));
5705 }
5706 }
5707 if fix.continuation.is_none() {
5708 fix.continuation = Some(ContinuationRecord::not_needed());
5709 }
5710 self.state.seats.insert(final_seat.key.clone(), final_seat);
5711
5712 let rescue_message = format!(
5713 "magi: operator-selected fix ({}) (uncommitted work)",
5714 self.state.operator_fixes[request_index]
5715 .findings
5716 .iter()
5717 .map(|f| f.id.as_str())
5718 .collect::<Vec<_>>()
5719 .join(", ")
5720 );
5721 let repo = self.state.repo.clone();
5722 let coordination = self.coordinate(&repo).await?;
5723 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5724 self.state.note_withheld("fix", &r.withheld);
5725 }
5726 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5727 fix.committed = after != head_at_request;
5728 git::worktree_remove(&self.state.repo, &fix_worktree)
5729 .await
5730 .ok();
5731 drop(coordination);
5732
5733 self.state.event(
5734 "fix",
5735 match &fix.failed {
5736 Some(reason) => format!(
5737 "operator fix: adoption report was lost ({reason}); {}",
5738 if fix.committed {
5739 "committed"
5740 } else {
5741 "NO new commit"
5742 }
5743 ),
5744 None => format!(
5745 "operator fix: {} addressed, {} rejected, {}",
5746 fix.addressed.len(),
5747 fix.rejected.len(),
5748 if fix.committed {
5749 "committed"
5750 } else {
5751 "NO new commit"
5752 }
5753 ),
5754 },
5755 );
5756
5757 for f in &mut self.state.operator_fixes[request_index].findings {
5764 f.outcome = if fix.failed.is_some() {
5765 OperatorFixOutcome::Unreported
5766 } else if fix.addressed.contains(&f.id) {
5767 OperatorFixOutcome::Addressed
5768 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5769 OperatorFixOutcome::Rejected { why: r.why.clone() }
5770 } else {
5771 OperatorFixOutcome::Unreported
5772 };
5773 }
5774
5775 let committed = fix.committed;
5776 if committed {
5777 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5778 }
5779 self.state.operator_fixes[request_index].fix = Some(fix);
5780 self.state.save()?;
5783
5784 if committed {
5785 self.state.event(
5786 "fix",
5787 format!(
5788 "operator fix committed {}; opening a follow-up review-only run",
5789 short(&after)
5790 ),
5791 );
5792 let origin =
5795 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5796 match Self::review(
5797 &self.state.repo,
5798 &winner.branch,
5799 self.state.config.clone(),
5800 origin,
5801 )
5802 .await
5803 {
5804 Ok(mut follow_up) => {
5805 follow_up.state.event(
5806 "start",
5807 format!(
5808 "requested by an operator fix on run {} for finding(s) {}",
5809 self.state.id,
5810 self.state.operator_fixes[request_index]
5811 .findings
5812 .iter()
5813 .map(|f| f.id.as_str())
5814 .collect::<Vec<_>>()
5815 .join(", "),
5816 ),
5817 );
5818 follow_up.state.save()?;
5819 let follow_up_id = follow_up.state.id.clone();
5820 let adopted = match crate::direct::adopt(
5823 &follow_up.state,
5824 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5825 ) {
5826 Ok(a) => a,
5827 Err(e) => {
5828 self.state.event(
5831 "fix",
5832 format!(
5833 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5834 ),
5835 );
5836 self.state.save()?;
5837 return Ok(());
5838 }
5839 };
5840 let executed = follow_up.execute().await;
5841 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5842 if let Some(a) = adopted {
5843 a.finish(&follow_up.state, executed);
5844 }
5845 if let Some(e) = failure {
5846 self.state.event(
5847 "fix",
5848 format!(
5849 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5850 ),
5851 );
5852 }
5853 self.state.operator_fixes[request_index].follow_up_review_run =
5854 Some(follow_up_id);
5855 }
5856 Err(e) => {
5857 self.state.event(
5858 "fix",
5859 format!("committed the fix but could not open a follow-up review: {e:#}"),
5860 );
5861 }
5862 }
5863 self.state.save()?;
5864 }
5865
5866 Ok(())
5867 }
5868
5869 async fn ask_fixer(
5889 &mut self,
5890 winner: &Candidate,
5891 node: &'static str,
5892 round: Option<usize>,
5893 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5894 ) -> (SeatJob, SeatState, AgentOutcome) {
5895 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5896 self.ask_fixer_among(attempts, node, round, None, build)
5897 .await
5898 }
5899
5900 async fn ask_fixer_among(
5905 &mut self,
5906 attempts: Vec<(AgentSpec, String)>,
5907 node: &'static str,
5908 round: Option<usize>,
5909 restore: Option<(&Path, &str)>,
5910 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5911 ) -> (SeatJob, SeatState, AgentOutcome) {
5912 let run_id = self.state.id.clone();
5913 let prompts = self.state.config.prompts.clone();
5914 let cache = self.state.config.cache_dir();
5915 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5916 let mut last = None;
5917 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5918 let seat = self.seat(&key, &spec.id);
5919 let mut job = build(spec.clone(), seat);
5920 if i > 0 {
5921 job.stem = format!("{}-{}", job.stem, spec.id);
5922 }
5923 let ctx = WaveCtx {
5924 carry_seats: false,
5925 pauses: Some((&self.pause, &self.interrupt)),
5926 run: &run_id,
5927 node,
5928 prompts: &prompts,
5929 cache: cache.as_deref(),
5930 round,
5931 };
5932 let (seat, out) =
5933 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5934 let advances = match &out {
5937 AgentOutcome::Ok(o) => agent::output_advances(o),
5938 AgentOutcome::ResourceBlocked(_) => false,
5939 _ => true,
5940 };
5941 if let Some(next) = ids.get(i + 1)
5942 && advances
5943 {
5944 self.state.seats.insert(seat.key.clone(), seat.clone());
5945 let class =
5946 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5947 let reason = match &out {
5948 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5949 other => fail_reason(other),
5950 };
5951 if let Some((wt, rev)) = restore
5955 && let Err(e) = self.graft_restore(wt, rev).await
5956 {
5957 let why =
5958 format!("could not restore the worktree before the next fixer: {e:#}");
5959 tracing::warn!("{why}");
5960 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5961 AgentOutcome::ResourceBlocked(why)
5962 } else {
5963 AgentOutcome::Failed(why)
5964 };
5965 last = Some((job, seat, out));
5966 break;
5967 }
5968 record_handover(
5969 &mut self.state,
5970 node,
5971 &seat.key,
5972 &spec.id,
5973 next,
5974 &class,
5975 &reason,
5976 );
5977 continue;
5978 }
5979 last = Some((job, seat, out));
5980 break;
5981 }
5982 last.expect("the fixer chain always has an entry")
5983 }
5984
5985 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5996 use crate::recovery::{Mode, Repair};
5997 if !self.state.reviews.is_empty() {
5998 return Ok(());
5999 }
6000 let Some(mut rec) = self.state.recovery.clone() else {
6001 return Ok(());
6002 };
6003 if rec.mode != Mode::VerifyRepair {
6004 return Ok(());
6005 }
6006 let wt = winner.worktree.clone();
6007 let mut head = git::rev_parse(&wt, "HEAD").await?;
6008 match rec.repair.clone() {
6009 Repair::Done { .. } => return Ok(()),
6010 Repair::WaitingForRestore { before } => {
6011 self.graft_restore(&wt, &before).await?;
6012 head = before;
6013 rec.repair = Repair::NotStarted;
6014 self.state.recovery = Some(rec.clone());
6015 self.state.save()?;
6016 }
6017 Repair::Started { before } => {
6018 if let Err(e) = self.graft_restore(&wt, &before).await {
6022 if e.is::<crate::repo_lock::ResourceBlocked>() {
6023 return Err(e);
6024 }
6025 return self
6026 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6027 .await;
6028 }
6029 rec.repair = Repair::Done {
6030 before: before.clone(),
6031 after: before,
6032 committed: false,
6033 failed: Some("interrupted; rolled back".to_owned()),
6034 };
6035 self.state
6036 .event("recovery", "the repair was interrupted and rolled back");
6037 self.state.recovery = Some(rec);
6038 self.state.save()?;
6039 return Ok(());
6040 }
6041 Repair::NotStarted => {}
6042 }
6043 if crate::recovery::recheck(&mut rec, &head) {
6044 self.state.event("recovery", rec.summary());
6045 self.state.recovery = Some(rec);
6046 self.state.save()?;
6047 return Ok(());
6048 }
6049 rec.repair = Repair::Started {
6050 before: head.clone(),
6051 };
6052 self.state.recovery = Some(rec.clone());
6053 self.state.save()?;
6054
6055 let language = self.state.config.graph.language.clone();
6056 let sessions = self.state.config.graph.sessions;
6057 let artifacts = agent::artifacts_dir(&self.state.dir());
6058 let fix_prompt = prompt::recovery_fix(
6059 &self.state.instruction,
6060 &rec,
6061 self.state.config.graph.review_rounds,
6062 &language,
6063 );
6064 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6065 self.state.event(
6066 "recovery",
6067 format!(
6068 "repair: the fixer starts from the failed verification on {}",
6069 short(&head)
6070 ),
6071 );
6072 let (_job, seat, out) = self
6073 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6074 prompt: fix_prompt.clone(),
6075 spec,
6076 seat,
6077 cwd: wt.clone(),
6078 timeout,
6079 allow_write: true,
6080 sessions,
6081 artifacts: artifacts.clone(),
6082 stem: "fix-recovery".to_owned(),
6083 handover: None,
6084 })
6085 .await;
6086 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6087 rec.repair = Repair::WaitingForRestore { before: head };
6088 self.state.recovery = Some(rec);
6089 return self.stop_for_resource();
6090 }
6091 let failed = match &out {
6092 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6093 AgentOutcome::Quota(o) => {
6094 self.state.quota.push(QuotaLoss {
6095 seat: seat.key.clone(),
6096 node: "fix".to_owned(),
6097 at: Timestamp::now(),
6098 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6099 });
6100 Some("rate limited (quota); the fixer could not run".to_owned())
6101 }
6102 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6103 AgentOutcome::Failed(e) => Some(e.clone()),
6104 };
6105 self.state.seats.insert(seat.key.clone(), seat);
6106 let coordination = self.coordinate(&wt).await?;
6107 let failed = match failed {
6108 Some(why) => {
6110 if let Err(e) = git::restore_to(&wt, &head).await {
6111 return self
6112 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6113 .await;
6114 }
6115 Some(why)
6116 }
6117 None => {
6118 if let Ok(r) = git::rescue_commit(
6119 &wt,
6120 "magi: recovery repair of the failed verification (uncommitted work)",
6121 )
6122 .await
6123 {
6124 self.state.note_withheld("fix", &r.withheld);
6125 }
6126 None
6127 }
6128 };
6129 let after = git::rev_parse(&wt, "HEAD").await?;
6130 drop(coordination);
6131 self.state.event(
6132 "recovery",
6133 match &failed {
6134 Some(why) => format!("repair: nothing kept ({why})"),
6135 None if after != head => {
6136 format!("repair: committed {}", short(&after))
6137 }
6138 None => "repair: the fixer made no commit".to_owned(),
6139 },
6140 );
6141 rec.repair = Repair::Done {
6142 before: head.clone(),
6143 committed: after != head,
6144 after,
6145 failed,
6146 };
6147 self.state.recovery = Some(rec);
6148 self.state.save()?;
6149 Ok(())
6150 }
6151
6152 async fn stop_recovery(
6155 &mut self,
6156 mut rec: crate::recovery::RecoveryRecord,
6157 head: &str,
6158 why: String,
6159 ) -> Result<()> {
6160 rec.repair = crate::recovery::Repair::Started {
6162 before: head.to_owned(),
6163 };
6164 self.state.recovery = Some(rec);
6165 self.state
6166 .event("recovery", format!("repair could not be undone: {why}"));
6167 self.state.status = RunStatus::Blocked;
6168 self.state.save()?;
6169 bail!("recovery repair could not be rolled back: {why}")
6170 }
6171
6172 async fn review_loop(&mut self) -> Result<()> {
6173 if self.hold_base_sync().await? {
6174 return Ok(());
6175 }
6176 loop {
6177 if self
6178 .state
6179 .base_sync
6180 .as_ref()
6181 .is_some_and(|s| s.conflict.is_some())
6182 {
6183 return Ok(());
6184 }
6185 if self.catchup_repairable() {
6186 let record = self.state.reviews.last().expect("repairable round").clone();
6187 let winner = self
6188 .state
6189 .winner()
6190 .cloned()
6191 .context("catch-up repair has no winner")?;
6192 if !self.fix_review_round(&winner, record).await? {
6193 return Ok(());
6194 }
6195 }
6196 self.review_loop_pass().await?;
6197 if !self.catchup_repairable() {
6198 return Ok(());
6199 }
6200 }
6201 }
6202
6203 fn catchup_repairable(&self) -> bool {
6206 self.state.reviews.len() < self.state.config.graph.review_rounds
6207 && self.state.reviews.last().is_some_and(|r| {
6208 !r.clean
6209 && r.blocking == 0
6210 && r.fix.is_none()
6211 && r.e2e_status() == E2eStatus::Failed
6212 })
6213 }
6214
6215 async fn review_loop_pass(&mut self) -> Result<()> {
6216 if self
6221 .state
6222 .base_sync
6223 .as_ref()
6224 .is_some_and(|s| s.conflict.is_some())
6225 {
6226 return Ok(());
6227 }
6228 let run_id = self.state.id.clone();
6233 let prompts = self.state.config.prompts.clone();
6234 let Some(winner) = self.state.winner().cloned() else {
6235 return Ok(());
6236 };
6237 let max_rounds = self.state.config.graph.review_rounds;
6238 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6248 if status == RunStatus::Gating {
6251 self.record_contested_handoff();
6252 }
6253 self.state.status = status;
6254 self.state.save()?;
6255 return Ok(());
6256 }
6257 self.state.status = RunStatus::Reviewing;
6258 if self.state.reviews.last().is_some_and(|r| {
6268 r.e2e_status() == E2eStatus::ResourceBlocked
6269 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6270 }) {
6271 let shell = self.state.config.shell();
6272 return self
6273 .stop_unresolved_review(
6274 "the last round's own verification never resolved",
6275 &shell,
6276 &winner.worktree,
6277 )
6278 .await;
6279 }
6280
6281 self.recovery_repair(&winner).await?;
6282 if self.resource_halt()? {
6283 return Ok(());
6284 }
6285
6286 let repo = self.state.repo.clone();
6287 let root = self.state.worktree_root();
6288 let language = self.state.config.graph.language.clone();
6289 let sessions = self.state.config.graph.sessions;
6290 let artifacts = self.merge_repair_artifacts();
6291 let base = self.landing_base();
6292 let base_short = short(&base);
6293 let reviewers = self.roles.reviewers.clone();
6294 let shell = self.state.config.shell();
6295
6296 for round in (self.state.reviews.len() + 1)..=max_rounds {
6297 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6298 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6299 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6300 let prev_verification = self
6309 .state
6310 .reviews
6311 .last()
6312 .and_then(|r| r.verification_summary(&head));
6313 let prior_section = match (&self.state.recovery, round) {
6316 (Some(rec), 1) if self.state.reviews.is_empty() => {
6317 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6318 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6319 Some(crate::recovery::head_of(
6320 &git::diff(&winner.worktree, reviewed, "HEAD")
6321 .await
6322 .unwrap_or_default(),
6323 crate::recovery::SINCE_DIFF_MAX,
6324 ))
6325 }
6326 _ => None,
6327 };
6328 Some(prompt::recovery_review(rec, since.as_deref()))
6329 }
6330 _ => None,
6331 };
6332
6333 let mut jobs = Vec::new();
6337 let coordination = self.coordinate(&repo).await?;
6338 for (r, spec) in reviewers.iter().cloned().enumerate() {
6339 let wt = root.join(format!("review-{}", r + 1));
6340 if wt.exists() {
6343 git::reset_detached(&wt, &head).await?;
6344 } else {
6345 git::worktree_add_detached(&repo, &wt, &head).await?;
6346 }
6347 self.setup_seat_worktree(&repo, &wt, false).await?;
6348 let seat_key = format!("review-{}", r + 1);
6349 let spec = pick_start_spec(
6352 &self.roles.reviewer_roster,
6353 spec,
6354 self.state.seat_history.get(&seat_key),
6355 );
6356 let seat = self.seat(&seat_key, &spec.id);
6357 jobs.push(SeatJob {
6358 prompt: prompt::review(&prompt::ReviewCtx {
6359 instruction: &self.state.instruction,
6360 branch: &winner.branch,
6361 base_short: &base_short,
6362 stat: &stat,
6363 patch: &patch,
6364 verification: prev_verification.as_ref(),
6365 prior: prior_section.as_deref(),
6366 reviewers: reviewers.len(),
6367 round,
6368 rounds: max_rounds,
6369 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6372 lens: Lens::for_seat(r),
6373 language: &language,
6374 github_language: &self.state.config.graph.github_language,
6375 }),
6376 spec,
6377 seat,
6378 cwd: wt,
6379 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6380 allow_write: false,
6381 sessions,
6382 artifacts: artifacts.clone(),
6383 stem: format!("review-{round}-{}", r + 1),
6384 handover: None,
6385 });
6386 }
6387
6388 drop(coordination);
6389 self.state.event(
6390 "review",
6391 format!(
6392 "round {round}: {} reviewers on {}",
6393 jobs.len(),
6394 short(&head)
6395 ),
6396 );
6397 let mut quota_losses = Vec::new();
6398 let review_retries = self.state.config.graph.retries;
6399 let review_cache = self.state.config.cache_dir();
6400 let ctx = WaveCtx {
6401 carry_seats: true,
6402 pauses: Some((&self.pause, &self.interrupt)),
6403 run: &run_id,
6404 node: "review",
6405 prompts: &prompts,
6406 cache: review_cache.as_deref(),
6407 round: Some(round),
6408 };
6409 let results = ask_json_wave::<Review>(
6410 jobs,
6411 Arc::clone(&self.sem),
6412 review_retries,
6413 &self.roles.reviewer_roster,
6414 &ctx,
6415 &mut quota_losses,
6416 &mut self.state,
6417 &|_: &Review| Ok(()),
6418 )
6419 .await;
6420 if self.state.resource_block.is_some() {
6421 return self.stop_for_resource();
6422 }
6423 let round_quota_missing = quota_losses.len();
6427 self.state.quota.extend(quota_losses);
6428
6429 let mut records = Vec::new();
6430 let mut all_findings = Vec::new();
6431 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6432 let agent_id = seat.agent.clone();
6433 self.state.seats.insert(seat.key.clone(), seat);
6434 let mut record = ReviewRecord {
6435 reviewer: r + 1,
6436 agent: agent_id,
6437 summary: String::new(),
6438 findings: Vec::new(),
6439 vote: None,
6440 failed: None,
6441 duration_ms: 0,
6442 attempts,
6448 };
6449 match res {
6450 Ok((review, out)) => {
6451 record.summary =
6459 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6460 record.vote = Some(review.vote);
6461 record.duration_ms = out.duration_ms;
6462 for (n, mut f) in review.findings.into_iter().enumerate() {
6463 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6466 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6467 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6468 f.file = f
6474 .file
6475 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6476 all_findings.push(f.clone());
6477 record.findings.push(f);
6478 }
6479 self.state.event(
6480 "review",
6481 format!(
6482 "round {round}: reviewer {} voted {} with {} finding(s)",
6483 r + 1,
6484 review.vote.label(),
6485 record.findings.len()
6486 ),
6487 );
6488 }
6489 Err(e) => {
6490 record.failed = Some(e.to_string());
6491 self.state.event(
6492 "review",
6493 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6494 );
6495 }
6496 }
6497 records.push(record);
6498 }
6499
6500 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6507 let vote_split =
6508 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6509 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6510 if vote_split {
6511 self.state.event(
6512 "review",
6513 format!(
6514 "round {round}: votes split ({}) — one round of reconsideration",
6515 initial_votes
6516 .iter()
6517 .map(|v| v.label())
6518 .collect::<Vec<_>>()
6519 .join(", ")
6520 ),
6521 );
6522 let panel: Vec<ReviewSeatReport<'_>> = records
6525 .iter()
6526 .filter_map(|r| {
6527 r.vote.map(|vote| ReviewSeatReport {
6528 reviewer: r.reviewer,
6529 vote,
6530 summary: &r.summary,
6531 findings: &r.findings,
6532 })
6533 })
6534 .collect();
6535
6536 let mut jobs = Vec::new();
6537 let mut seats_at = Vec::new();
6538 for (r, spec) in reviewers.iter().cloned().enumerate() {
6539 if records[r].vote.is_none() {
6543 continue;
6544 }
6545 let wt = root.join(format!("review-{}", r + 1));
6546 let seat_key = format!("review-{}", r + 1);
6547 let spec = self.occupant(&seat_key, spec);
6548 let seat = self.seat(&seat_key, &spec.id);
6549 let build = |with_patch: bool| {
6557 prompt::review_reconsider(&ReviewReconsiderCtx {
6558 instruction: &self.state.instruction,
6559 reviewer: r + 1,
6560 lens: Lens::for_seat(r),
6561 panel: &panel,
6562 patch: with_patch.then_some(ReviewPatch {
6563 branch: &winner.branch,
6564 base_short: &base_short,
6565 stat: &stat,
6566 patch: &patch,
6567 }),
6568 round,
6569 rounds: max_rounds,
6570 language: &language,
6571 })
6572 };
6573 let full = build(true);
6574 let prompt = if has_context(&spec, &seat, sessions) {
6575 build(false)
6576 } else {
6577 full.clone()
6578 };
6579 jobs.push(SeatJob {
6580 prompt,
6581 spec,
6582 seat,
6583 cwd: wt,
6584 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6585 allow_write: false,
6586 sessions,
6587 artifacts: artifacts.clone(),
6588 stem: format!("review-{round}-reconsider-{}", r + 1),
6589 handover: Some(full),
6590 });
6591 seats_at.push(r);
6592 }
6593
6594 let mut recon_quota_losses = Vec::new();
6595 let recon_cache = self.state.config.cache_dir();
6596 let recon_ctx = WaveCtx {
6597 carry_seats: true,
6598 pauses: Some((&self.pause, &self.interrupt)),
6599 run: &run_id,
6600 node: "review",
6601 prompts: &prompts,
6602 cache: recon_cache.as_deref(),
6603 round: Some(round),
6604 };
6605 let recon_results = ask_json_wave::<ReviewRevote>(
6606 jobs,
6607 Arc::clone(&self.sem),
6608 review_retries,
6609 &self.roles.reviewer_roster,
6610 &recon_ctx,
6611 &mut recon_quota_losses,
6612 &mut self.state,
6613 &|_: &ReviewRevote| Ok(()),
6614 )
6615 .await;
6616 if self.state.resource_block.is_some() {
6617 return self.stop_for_resource();
6618 }
6619 self.state.quota.extend(recon_quota_losses);
6620
6621 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6622 let agent_id = seat.agent.clone();
6623 self.state.seats.insert(seat.key.clone(), seat);
6624 let mut rec = ReviewRevoteRecord {
6625 reviewer: r + 1,
6626 agent: agent_id,
6627 vote: None,
6628 reason: String::new(),
6629 failed: None,
6630 };
6631 match res {
6632 Ok((rv, _)) => {
6633 rec.vote = Some(rv.vote);
6634 rec.reason =
6635 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6636 self.state.event(
6637 "review",
6638 format!(
6639 "round {round}: reviewer {} revoted {}",
6640 r + 1,
6641 rv.vote.label()
6642 ),
6643 );
6644 }
6645 Err(e) => {
6646 rec.failed = Some(e.to_string());
6647 self.state.event(
6648 "review",
6649 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6650 );
6651 }
6652 }
6653 reconsideration.push(rec);
6654 }
6655 } else if initial_votes.len() > 1 {
6656 self.state.event(
6657 "review",
6658 format!(
6659 "round {round}: votes agreed ({}) — no reconsideration",
6660 initial_votes[0].label()
6661 ),
6662 );
6663 }
6664
6665 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6666 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6667 let defer_e2e =
6678 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6679 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6680 let reason =
6681 format!("{blocking} blocking finding(s) already required a fix this round");
6682 self.state.event(
6683 "verify",
6684 format!(
6685 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6686 {}); it will run once a round has none left",
6687 short(&head)
6688 ),
6689 );
6690 (Vec::new(), false, true, Some(reason))
6691 } else {
6692 let e2e_commands = self.state.config.verify.e2e.clone();
6693 let cache_dir = self.state.config.cache_dir();
6694 let context = format!("round {round}");
6695 let wait = self.repository_wait();
6696 let (e2e, verify_retried) = with_cache_lease(
6697 &mut self.state,
6698 cache_dir.as_deref(),
6699 "e2e",
6700 "e2e",
6701 &winner.worktree,
6702 &head,
6703 verify_timeout,
6704 &context,
6705 &wait,
6706 |state, budget| {
6707 let shell = shell.clone();
6708 let e2e_commands = e2e_commands.clone();
6709 let worktree = winner.worktree.clone();
6710 let context = context.clone();
6711 async move {
6712 run_e2e_with_retry(
6713 state,
6714 &shell,
6715 &e2e_commands,
6716 &worktree,
6717 budget,
6718 &context,
6719 )
6720 .await
6721 }
6722 },
6723 )
6724 .await?;
6725 (e2e, verify_retried, false, None)
6726 };
6727
6728 let expected = records.len();
6729 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6730 let incomplete = answered < expected;
6731 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6732 let policy = self.state.config.graph.incomplete_review;
6733 let clean = round_is_clean(
6734 blocking,
6735 e2e_ok,
6736 answered,
6737 expected,
6738 round_quota_missing,
6739 policy,
6740 );
6741
6742 let mut round_record = ReviewRound {
6743 round,
6744 head: head.clone(),
6745 verified_head: None,
6746 verified_at: None,
6747 reviews: records,
6748 e2e,
6749 verify_retried,
6750 e2e_deferred,
6751 e2e_defer_reason,
6752 fix: None,
6753 blocking,
6754 answered,
6755 expected,
6756 clean,
6757 progressed: false,
6758 vote_split,
6759 reconsideration,
6760 verdict: None,
6761 };
6762 round_record.verdict = ReviewVote::worst(
6766 round_record
6767 .final_votes()
6768 .into_iter()
6769 .map(|(_, _, vote)| vote),
6770 );
6771 if !matches!(
6782 round_record.e2e_status(),
6783 E2eStatus::Deferred | E2eStatus::NotConfigured
6784 ) {
6785 round_record.verified_head = Some(head.clone());
6786 round_record.verified_at = Some(Timestamp::now());
6787 }
6788
6789 if incomplete {
6790 let missing: Vec<String> = round_record
6791 .reviews
6792 .iter()
6793 .filter(|r| r.failed.is_some())
6794 .map(|r| format!("review-{}", r.reviewer))
6795 .collect();
6796 self.state.event(
6797 "review",
6798 format!(
6799 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6800 missing.join(", ")
6801 ),
6802 );
6803 }
6804
6805 if clean {
6806 self.state.event(
6807 "review",
6808 if incomplete && policy == IncompleteReviewPolicy::Warn {
6809 format!(
6810 "round {round}: clean (warn policy, incomplete panel) — no \
6811 blocking findings from the seats that answered, verification green"
6812 )
6813 } else if incomplete {
6814 format!(
6815 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6816 quorum) — no blocking findings from the seats that answered, \
6817 verification green",
6818 expected - answered
6819 )
6820 } else {
6821 format!("round {round}: clean — no blocking findings, verification green")
6822 },
6823 );
6824 self.state.reviews.push(round_record);
6825 self.state.status = RunStatus::Gating;
6826 self.state.save()?;
6827 return Ok(());
6828 }
6829
6830 if incomplete && blocking == 0 && e2e_ok {
6838 self.state.reviews.push(round_record);
6839 self.state.save()?;
6840 if round == max_rounds {
6841 self.state.status = RunStatus::Blocked;
6842 self.state.event(
6843 "review",
6844 format!(
6845 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6846 refusing to call it clean",
6847 expected - answered
6848 ),
6849 );
6850 return Ok(());
6851 }
6852 continue;
6853 }
6854
6855 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6867 self.state.reviews.push(round_record);
6868 return self
6869 .stop_unresolved_review(
6870 "the round's own verification could not run",
6871 &shell,
6872 &winner.worktree,
6873 )
6874 .await;
6875 }
6876
6877 if round == max_rounds {
6878 self.state.reviews.push(round_record);
6879 return self
6880 .stop_reviewing(
6881 &format!(
6882 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6883 ),
6884 &shell,
6885 &winner.worktree,
6886 )
6887 .await;
6888 }
6889
6890 if !self.fix_review_round(&winner, round_record).await? {
6891 return Ok(());
6892 }
6893 }
6894 Ok(())
6895 }
6896
6897 async fn fix_review_round(
6900 &mut self,
6901 winner: &Candidate,
6902 mut round_record: ReviewRound,
6903 ) -> Result<bool> {
6904 let round = round_record.round;
6905 let shell = self.state.config.shell();
6906 let step = self.run_fix_round(winner, &mut round_record).await?;
6907 if matches!(step, FixStep::Resource) {
6908 self.stop_for_resource()?;
6909 return Ok(false);
6910 }
6911 if self.state.reviews.len() < round {
6912 self.state.reviews.push(round_record);
6913 } else {
6914 self.state.reviews[round - 1] = round_record;
6915 }
6916 self.state.save()?;
6917
6918 if matches!(step, FixStep::Recorded { report_lost: true }) {
6931 self.stop_reviewing(
6932 "the fixer's adoption report never came back, even after resuming its \
6933 own seat; refusing to start another round against the same worktree \
6934 while that is unresolved",
6935 &shell,
6936 &winner.worktree,
6937 )
6938 .await?;
6939 return Ok(false);
6940 }
6941
6942 let streak = self
6943 .state
6944 .reviews
6945 .iter()
6946 .rev()
6947 .take_while(|r| !r.progressed)
6948 .count();
6949 if streak >= STAGNANT_LIMIT {
6950 self.stop_reviewing(
6951 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6952 &shell,
6953 &winner.worktree,
6954 )
6955 .await?;
6956 return Ok(false);
6957 }
6958 Ok(true)
6959 }
6960
6961 async fn run_fix_round(
6963 &mut self,
6964 winner: &Candidate,
6965 round_record: &mut ReviewRound,
6966 ) -> Result<FixStep> {
6967 let round = round_record.round;
6968 let max_rounds = self.state.config.graph.review_rounds;
6969 let run_id = self.state.id.clone();
6970 let prompts = self.state.config.prompts.clone();
6971 let language = self.state.config.graph.language.clone();
6972 let sessions = self.state.config.graph.sessions;
6973 let artifacts = self.merge_repair_artifacts();
6974 let base = self.landing_base();
6975 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6976 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6977 let this_round_verification = round_record.verification_summary(&head);
6978 let all_findings: Vec<_> = round_record
6979 .reviews
6980 .iter()
6981 .flat_map(|r| r.findings.iter().cloned())
6982 .collect();
6983 let blocking_findings: Vec<_> = all_findings
6986 .iter()
6987 .filter(|f| f.severity.blocks())
6988 .cloned()
6989 .collect();
6990 let fix_prompt = prompt::fix_in(
6991 &self.state.instruction,
6992 &blocking_findings,
6993 this_round_verification.as_ref(),
6994 round,
6995 max_rounds,
6996 &language,
6997 &self.state.config.graph.github_language,
6998 );
6999 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7000 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7001 let (job, seat, out) = self
7002 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7003 prompt: fix_prompt.clone(),
7004 spec,
7005 seat,
7006 cwd: winner.worktree.clone(),
7007 timeout,
7008 allow_write: true,
7009 sessions,
7010 artifacts: artifacts.clone(),
7011 stem: format!("fix-{round}"),
7012 handover: None,
7013 })
7014 .await;
7015 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7018 return Ok(FixStep::Resource);
7019 }
7020 let agent_id = seat.agent.clone();
7021
7022 let mut fix = FixRecord {
7023 agent: agent_id,
7024 addressed: Vec::new(),
7025 rejected: Vec::new(),
7026 notes: String::new(),
7027 committed: false,
7028 failed: None,
7029 duration_ms: 0,
7030 continuation: None,
7031 };
7032 let mut continuation = ContinuationRecord::not_needed();
7033 let mut final_seat = seat.clone();
7034 match out {
7035 AgentOutcome::Ok(o) => {
7036 fix.duration_ms = o.duration_ms;
7037 let parsed = verdict::extract_json::<FixReport>(&o.text);
7038 let incomplete_reason = match &parsed {
7045 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7046 "the reply parsed, but it reported a command whose own CLI never \
7047 confirmed an exit status"
7048 .to_owned(),
7049 ),
7050 Ok(_) => None,
7051 Err(e) => Some(e.to_string()),
7052 };
7053 match incomplete_reason {
7054 None => {
7055 let report = parsed.expect("checked Ok above");
7056 fix.addressed = report.addressed;
7057 fix.rejected = report.rejected;
7058 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7059 }
7060 Some(reason) => {
7061 let (resumed_seat, resolved, failure, cont) = self
7062 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7063 .await;
7064 fix.duration_ms += cont.cumulative_wait_ms;
7065 continuation = cont;
7066 final_seat = resumed_seat;
7067 match resolved {
7068 Some(report) => {
7069 fix.addressed = report.addressed;
7070 fix.rejected = report.rejected;
7071 fix.notes =
7072 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7073 }
7074 None => fix.failed = failure,
7075 }
7076 }
7077 }
7078 }
7079 AgentOutcome::Dropped(o) => {
7081 fix.duration_ms = o.duration_ms;
7082 let why = o
7083 .dropped
7084 .as_ref()
7085 .map(|d| d.why.as_str())
7086 .unwrap_or("the CLI ended the stream without delivering its answer");
7087 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7088 }
7089 AgentOutcome::Quota(o) => {
7090 self.state.quota.push(QuotaLoss {
7091 seat: final_seat.key.clone(),
7092 node: "fix".to_owned(),
7093 at: Timestamp::now(),
7094 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7095 });
7096 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7097 }
7098 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7099 }
7100 fix.continuation = Some(continuation);
7101 self.state.seats.insert(final_seat.key.clone(), final_seat);
7102 let coordination = self.coordinate(&winner.worktree).await?;
7103 if let Ok(r) = git::rescue_commit(
7104 &winner.worktree,
7105 &format!("magi: review round {round} fixes (uncommitted work)"),
7106 )
7107 .await
7108 {
7109 self.state.note_withheld("fix", &r.withheld);
7110 }
7111 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7112 fix.committed = after != before;
7113 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7121 drop(coordination);
7122 let progressed = diff_after != patch;
7123 let commit_note = if fix.committed {
7124 "committed"
7125 } else {
7126 "NO new commit"
7127 };
7128 let tree_note = if progressed {
7129 "changed vs base"
7130 } else {
7131 "unchanged vs base"
7132 };
7133 self.state.event(
7134 "fix",
7135 match &fix.failed {
7136 Some(reason) => {
7142 format!(
7143 "round {round}: fixer's adoption report was lost ({reason}); \
7144 {commit_note}, tree {tree_note}"
7145 )
7146 }
7147 None => format!(
7148 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7149 {tree_note}{}",
7150 fix.addressed.len(),
7151 fix.rejected.len(),
7152 if continuation.outcome == ContinuationOutcome::Resumed {
7153 format!(
7154 " (adoption report recovered after {} continuation(s))",
7155 continuation.attempts
7156 )
7157 } else {
7158 String::new()
7159 },
7160 ),
7161 },
7162 );
7163 round_record.fix = Some(fix);
7164 round_record.progressed = progressed;
7165 Ok(FixStep::Recorded {
7166 report_lost: matches!(
7167 continuation.outcome,
7168 ContinuationOutcome::Exhausted
7169 | ContinuationOutcome::QuotaLost
7170 | ContinuationOutcome::NoSession
7171 ),
7172 })
7173 }
7174
7175 fn record_contested_handoff(&mut self) {
7209 if self.state.contested_handoff.is_some() {
7210 return;
7211 }
7212 let Some(contested) = self
7213 .state
7214 .reviews
7215 .last()
7216 .and_then(ReviewRound::contested_handoff)
7217 else {
7218 return;
7219 };
7220 self.state.event(
7221 "review",
7222 format!(
7223 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7224 wait for the owner's approval",
7225 contested.findings.len(),
7226 contested.rejecters.len()
7227 ),
7228 );
7229 self.state.contested_handoff = Some(contested);
7230 }
7231
7232 async fn stop_unresolved_review(
7233 &mut self,
7234 why: &str,
7235 shell: &[String],
7236 worktree: &Path,
7237 ) -> Result<()> {
7238 self.stop_reviewing_inner(why, shell, worktree, true).await
7239 }
7240
7241 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7242 self.stop_reviewing_inner(why, shell, worktree, false).await
7243 }
7244
7245 async fn stop_reviewing_inner(
7246 &mut self,
7247 why: &str,
7248 shell: &[String],
7249 worktree: &Path,
7250 repairable: bool,
7251 ) -> Result<()> {
7252 let round_idx = self.state.reviews.len() - 1;
7253 let needs_catchup_run = matches!(
7261 self.state.reviews[round_idx].e2e_status(),
7262 E2eStatus::Deferred | E2eStatus::ResourceBlocked
7263 );
7264 if needs_catchup_run {
7265 let round = self.state.reviews[round_idx].round;
7266 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7267 let commands = self.state.config.verify.e2e.clone();
7268 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7269 let cache_dir = self.state.config.cache_dir();
7270 let context = format!(
7271 "round {round}: verification unresolved, catching up before the final decision"
7272 );
7273 let wait = self.repository_wait();
7274 let (outcomes, verify_retried) = with_cache_lease(
7275 &mut self.state,
7276 cache_dir.as_deref(),
7277 "e2e",
7278 "e2e",
7279 worktree,
7280 &attempted_head,
7281 timeout,
7282 &context,
7283 &wait,
7284 |state, budget| {
7285 let shell = shell.to_vec();
7286 let commands = commands.clone();
7287 let context = context.clone();
7288 async move {
7289 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7290 .await
7291 }
7292 },
7293 )
7294 .await?;
7295 let last = &mut self.state.reviews[round_idx];
7296 last.e2e = outcomes;
7297 last.verify_retried = verify_retried;
7298 last.verified_head = Some(attempted_head);
7305 last.verified_at = Some(Timestamp::now());
7306 if verify_inconclusive(&last.e2e) {
7307 self.state.save()?;
7314 return Ok(());
7315 }
7316 last.e2e_deferred = false;
7317 }
7318 let last = &self.state.reviews[round_idx];
7319 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7320
7321 match last.e2e_status() {
7322 E2eStatus::Failed => {
7323 let red: Vec<String> = last
7324 .e2e
7325 .iter()
7326 .filter(|o| !o.ok())
7327 .map(|o| {
7328 format!(
7329 "`{}` -> {:?}\n{}",
7330 o.command,
7331 o.code,
7332 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7333 )
7334 })
7335 .collect();
7336 let can_repair = last.blocking == 0 && last.fix.is_none();
7337 self.state.reviews[round_idx].clean = false;
7338 self.state
7339 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7340 self.state.status = RunStatus::Blocked;
7341 if repairable
7342 && can_repair
7343 && self.state.reviews.len() < self.state.config.graph.review_rounds
7344 {
7345 self.state.status = RunStatus::Reviewing;
7346 }
7347 }
7348 E2eStatus::ResourceBlocked => {
7353 self.state.event(
7354 "review",
7355 format!(
7356 "{why}; e2e could not run (resource unavailable); not \
7357 deciding yet"
7358 ),
7359 );
7360 }
7361 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7362 self.state.event(
7363 "review",
7364 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7365 );
7366 self.record_contested_handoff();
7367 self.state.status = RunStatus::Gating;
7368 }
7369 }
7370 self.state.save()?;
7371 Ok(())
7372 }
7373
7374 async fn gate(&mut self) -> Result<()> {
7377 if self.hold_base_sync().await? {
7378 return Ok(());
7379 }
7380 if self.state.status == RunStatus::Failed
7392 || self
7393 .state
7394 .base_sync
7395 .as_ref()
7396 .is_some_and(|s| s.conflict.is_some())
7397 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7398 != Some(RunStatus::Gating)
7399 {
7400 return Ok(());
7401 }
7402 let Some(winner) = self.state.winner().cloned() else {
7403 return Ok(());
7404 };
7405 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7406 self.state.reconcile_verification_head(&head);
7407 self.state.save()?;
7408 let wait = self.repository_wait();
7409 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7410 return Ok(());
7411 }
7412 if self.state.gate_ran {
7413 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7424 self.state.status = RunStatus::Blocked;
7425 self.state.save()?;
7426 }
7427 return Ok(());
7428 }
7429 self.state.status = RunStatus::Gating;
7430 let mut outcomes = self.run_gate(&winner).await?;
7431 let mut attempted_head = self.state.gate_head.clone();
7432 loop {
7433 if verify_inconclusive(&outcomes) {
7444 self.state.save()?;
7445 return Ok(());
7446 }
7447 if outcomes.iter().all(CommandOutcome::ok) {
7448 break;
7449 }
7450 let fixes_before = self.state.gate_fixes.len();
7451 match self.gate_fix_round(&winner, &outcomes).await? {
7452 GateFix::Retry => {
7453 outcomes = self.run_gate(&winner).await?;
7454 attempted_head = self.state.gate_head.clone();
7455 }
7456 GateFix::Stop => {
7457 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7458 if self.state.gate_fixes.len() > fixes_before
7459 && attempted_head.as_deref() != Some(current.as_str())
7460 && verify_current_head(&mut self.state, &winner, &wait).await?
7461 {
7462 outcomes = self.run_gate(&winner).await?;
7465 attempted_head = self.state.gate_head.clone();
7466 } else {
7467 break;
7468 }
7469 }
7470 GateFix::Defer => {
7471 self.state.save()?;
7472 return Ok(());
7473 }
7474 }
7475 }
7476 let passed = outcomes.iter().all(CommandOutcome::ok);
7477 self.state.gate = outcomes;
7478 self.state.gate_ran = true;
7479 self.state.gate_head = attempted_head;
7480 self.state.save()?;
7481 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7482 return Ok(());
7483 }
7484 if !passed {
7485 self.state.status = RunStatus::Blocked;
7486 let spent = self.state.gate_fixes.len();
7487 self.state.event(
7488 "gate",
7489 if spent == 0 {
7490 "gate failed; not merging".to_owned()
7491 } else {
7492 format!("gate failed after {spent} gate-fix round(s); not merging")
7493 },
7494 );
7495 }
7496 self.state.save()?;
7497 Ok(())
7498 }
7499
7500 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7510 let commands = state.config.verify.pre_gate.clone();
7511 if commands.is_empty() {
7512 return;
7513 }
7514 let shell = state.config.shell();
7515 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7516 let (outcomes, _) = run_commands(
7517 state,
7518 "pre_gate",
7519 "pre_gate",
7520 0,
7521 &shell,
7522 &commands,
7523 &winner.worktree,
7524 timeout,
7525 )
7526 .await;
7527 for o in &outcomes {
7528 if !o.ok() {
7529 tracing::warn!(
7530 "pre_gate `{}` failed ({:?}); the gate decides",
7531 o.command,
7532 o.code
7533 );
7534 }
7535 state.event(
7536 "pre_gate",
7537 format!(
7538 "`{}` -> {}",
7539 o.command,
7540 if o.ok() {
7541 "pass".to_owned()
7542 } else {
7543 format!(
7544 "FAIL ({:?})\n{}",
7545 o.code,
7546 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7547 )
7548 }
7549 ),
7550 );
7551 }
7552 state.pre_gate = outcomes;
7553 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7554 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7555 Ok(head) => {
7556 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7557 state.pre_gate_commit = Some(head);
7558 }
7559 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7560 },
7561 Ok(false) => {}
7562 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7563 }
7564 if let Err(e) = state.save() {
7565 tracing::warn!("could not persist the pre_gate record: {e:#}");
7566 }
7567 }
7568
7569 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7572 let wait = self.repository_wait();
7573 run_landing_gate(&mut self.state, winner, &wait).await
7574 }
7575
7576 async fn gate_fix_round(
7589 &mut self,
7590 winner: &Candidate,
7591 outcomes: &[CommandOutcome],
7592 ) -> Result<GateFix> {
7593 let cap = self.state.config.graph.gate_fix_rounds;
7594 let spent = self.state.gate_fixes.len();
7595 if spent >= cap {
7596 if cap > 0 {
7597 self.state.event(
7598 "gate",
7599 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7600 );
7601 }
7602 return Ok(GateFix::Stop);
7603 }
7604 if !gate_fixable(outcomes) {
7605 self.state.event(
7606 "gate",
7607 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7608 command or similar); not spending a fix round on it",
7609 );
7610 return Ok(GateFix::Stop);
7611 }
7612 let min_free = self.state.config.disk.min_free_bytes;
7613 if min_free > 0 {
7614 match crate::disk::free_bytes(&winner.worktree) {
7615 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7616 Ok(free) => {
7617 self.state.event(
7618 "gate",
7619 format!(
7620 "only {free} bytes free ({min_free} required by `[disk] \
7621 min_free_bytes`); not spending a fix round on a failure the disk \
7622 may explain"
7623 ),
7624 );
7625 return Ok(GateFix::Stop);
7626 }
7627 Err(e) => {
7628 self.state.event(
7629 "gate",
7630 format!("free disk space could not be measured ({e:#}); no fix round"),
7631 );
7632 return Ok(GateFix::Stop);
7633 }
7634 }
7635 }
7636
7637 let attempt = spent + 1;
7638 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7639 let base = self.landing_base();
7640 let fix_prompt = prompt::gate_fix_in(
7641 &self.state.instruction,
7642 &failed,
7643 attempt,
7644 cap,
7645 &self.state.config.graph.language,
7646 &self.state.config.graph.github_language,
7647 );
7648 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7649 let sessions = self.state.config.graph.sessions;
7650 let artifacts = self.merge_repair_artifacts();
7651 self.state.event(
7652 "gate",
7653 format!("gate failed; gate-fix round {attempt} of {cap}"),
7654 );
7655 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7656 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7657 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7660 head: before.clone(),
7661 passed: false,
7662 outcomes: Vec::new(),
7663 });
7664 self.state.save()?;
7665 let (_, seat, out) = self
7666 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7667 prompt: fix_prompt.clone(),
7668 spec,
7669 seat,
7670 cwd: winner.worktree.clone(),
7671 timeout,
7672 allow_write: true,
7673 sessions,
7674 artifacts: artifacts.clone(),
7675 stem: format!("gate-fix-{attempt}"),
7676 handover: None,
7677 })
7678 .await;
7679 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7682 return Ok(GateFix::Defer);
7683 }
7684 let mut record = GateFixRecord {
7685 agent: seat.agent.clone(),
7686 failed,
7687 notes: String::new(),
7688 committed: false,
7689 error: None,
7690 };
7691 match out {
7692 AgentOutcome::Ok(o) => {
7693 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7696 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7697 }
7698 }
7699 AgentOutcome::Dropped(_) => {
7700 record.error = Some("the CLI dropped the stream".to_owned());
7701 }
7702 AgentOutcome::Quota(o) => {
7703 self.state.quota.push(QuotaLoss {
7704 seat: seat.key.clone(),
7705 node: "gate-fix".to_owned(),
7706 at: Timestamp::now(),
7707 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7708 });
7709 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7710 }
7711 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7712 }
7713 self.state.seats.insert(seat.key.clone(), seat);
7714 let coordination = self.coordinate(&winner.worktree).await?;
7715 if let Ok(r) = git::rescue_commit(
7716 &winner.worktree,
7717 &format!("magi: gate fix {attempt} (uncommitted work)"),
7718 )
7719 .await
7720 {
7721 self.state.note_withheld("gate-fix", &r.withheld);
7722 }
7723 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7724 record.committed = after != before;
7725 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7726 let note = record.error.clone();
7727 self.state.gate_fixes.push(record);
7728 self.state.save()?;
7729 if !changed {
7730 self.state.event(
7731 "gate",
7732 match note {
7733 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7734 None => format!("gate-fix round {attempt}: the tree did not change"),
7735 },
7736 );
7737 return Ok(GateFix::Stop);
7738 }
7739 self.state.event(
7740 "gate",
7741 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7742 );
7743
7744 drop(coordination);
7745 let wait = self.repository_wait();
7746 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7747 }
7748
7749 async fn guarded_pr_message(
7756 &mut self,
7757 winner: &Candidate,
7758 facts: Option<&BranchFacts>,
7759 posting: bool,
7760 ) -> Result<Option<PrMessage>> {
7761 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7762 let head = match final_head.as_deref() {
7763 Some(h) => crate::pr_outcome::FinalHead::At(h),
7764 None => crate::pr_outcome::FinalHead::Unknown,
7765 };
7766 let summarized = if posting {
7768 Some(self.summarize_change(winner, final_head.as_deref()).await)
7769 } else {
7770 None
7771 };
7772 if self.state.resource_block.is_some() {
7777 self.state.event(
7778 "pr-summary",
7779 "a shared resource was busy; nothing was published, resume to summarize again",
7780 );
7781 return Ok(None);
7782 }
7783 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7784 anyhow::bail!(
7785 "the final commit moved during public description generation; retry before publishing"
7786 );
7787 }
7788 let note = match (&summarized, &self.state.change_summary) {
7789 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7790 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7791 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7792 (None, _) => ChangeNote::NotAsked,
7793 };
7794 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7795 if !posting {
7796 let identity = crate::scrub::Identity::current();
7797 return Ok(Some(PrMessage {
7798 title: crate::scrub::scrub(&pr.title, &identity),
7799 body: crate::scrub::scrub(&pr.body, &identity),
7800 }));
7801 }
7802 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7805 if let Some(known) = self
7806 .state
7807 .github_text
7808 .clone()
7809 .filter(|g| g.fingerprint == raw_fp)
7810 {
7811 return self.settle_withheld(winner, &pr, known).await;
7812 }
7813 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7814 let mut violations =
7815 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7816 let identity = crate::scrub::Identity::current();
7817 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7818 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7819 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7820 {
7821 violations.push(crate::github_text::Violation::SensitiveData);
7822 }
7823 let has_language = violations
7827 .iter()
7828 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7829 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7830 self.state.event(
7831 "github-text",
7832 if has_language {
7833 format!("description rejected: {violations:?}; requesting one rewrite")
7834 } else {
7835 format!("description rejected: {violations:?}; asking the owner")
7836 },
7837 );
7838 let mut rewritten = false;
7839 let mut rejected_rewrite = None;
7840 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7841 {
7842 let output = text;
7843 if let Some(title) = summary_title(&output) {
7844 let raw_body = summary_without_title(&output);
7847 let clean_title = crate::scrub::scrub(&title, &identity);
7848 let mut body = crate::scrub::scrub(&raw_body, &identity);
7849 let redacted = clean_title != title || body != raw_body;
7850 let title = clean_title;
7851 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7852 if !complete {
7853 self.state.event(
7854 "github-text",
7855 "description rewrite omitted required findings",
7856 );
7857 }
7858 if !body.trim().is_empty() {
7859 body.push_str("\n\n");
7860 body.push_str(&public_pr_evidence(&self.state, head));
7861 }
7862 let marker = format!("magi:run/{}", self.state.id);
7863 if !body.trim().is_empty() && !body.contains(&marker) {
7864 body.push_str(&format!("\n\n{marker}"));
7865 }
7866 let rewrite = PrMessage {
7867 title: title.clone(),
7868 body: body.clone(),
7869 };
7870 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7871 let mut failed = crate::github_text::check_for(
7872 &self.state.config.graph,
7873 &title,
7874 &body,
7875 redecision,
7876 );
7877 if (!crate::github_text::shareable(&title)
7878 || !crate::github_text::shareable(&body))
7879 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7880 {
7881 failed.push(crate::github_text::Violation::SensitiveData);
7882 }
7883 if complete
7884 && !title.trim().is_empty()
7885 && !body.trim().is_empty()
7886 && !failed.is_empty()
7887 {
7888 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7889 title: title.clone(),
7890 body: body.clone(),
7891 failed: failed
7892 .iter()
7893 .map(|v| crate::github_text::category(*v).to_owned())
7894 .collect(),
7895 });
7896 }
7897 if complete
7898 && !title.trim().is_empty()
7899 && !raw_body.trim().is_empty()
7900 && failed.is_empty()
7901 {
7902 decision = redecision;
7903 pr = PrMessage { title, body };
7904 rewritten = true;
7905 self.state
7906 .event("github-text", "description rewrite passed");
7907 if redacted {
7908 self.state.event(
7909 "github-text",
7910 "sensitive data removed from description rewrite",
7911 );
7912 }
7913 }
7914 }
7915 }
7916 if !rewritten {
7917 self.state.event(
7918 "github-text",
7919 "description rewrite unavailable or rejected; asking the owner",
7920 );
7921 if let Some(w) =
7922 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7923 {
7924 self.retire_github_text_question("the withheld text changed");
7927 let record = crate::run::GithubTextAsk {
7928 rewrite: rejected_rewrite,
7929 fingerprint: raw_fp,
7930 title: w.title,
7931 body: w.body,
7932 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7933 question: None,
7934 asks: 0,
7935 resolved: false,
7936 chosen_title: None,
7937 choice: None,
7938 };
7939 return self.settle_withheld(winner, &pr, record).await;
7940 }
7941 }
7942 }
7943 let (title, body) =
7944 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7945 Ok(Some(PrMessage { title, body }))
7946 }
7947
7948 async fn summarize_change(
7959 &mut self,
7960 winner: &Candidate,
7961 head: Option<&str>,
7962 ) -> std::result::Result<(), String> {
7963 let Some(head) = head else {
7964 return Err("the final commit could not be read".to_owned());
7965 };
7966 let remote = self.state.config.merge.remote.clone();
7967 let tracking = format!("{remote}/{}", self.state.base_branch);
7968 if !git::rev_exists(&winner.worktree, &tracking).await {
7969 return Err(format!("the remote base `{tracking}` could not be read"));
7970 }
7971 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7972 Ok(b) if !b.is_empty() => b,
7973 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7974 };
7975 let evidence = crate::pr_outcome::summary_evidence(
7976 &self.state,
7977 crate::pr_outcome::FinalHead::At(head),
7978 );
7979 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7980 if self
7981 .state
7982 .change_summary
7983 .as_ref()
7984 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7985 {
7986 return Ok(());
7987 }
7988 if self.state.change_summary.take().is_some() {
7989 self.state.event(
7990 "pr-summary",
7991 "the final diff or evidence changed; the earlier change summary was discarded",
7992 );
7993 }
7994 if let Err(e) = write_artifact(
7996 &self.state,
7997 &format!("public-description-{fingerprint}.md"),
7998 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7999 ) {
8000 tracing::warn!("could not save public description evidence: {e:#}");
8001 }
8002 let chain = match agent::pick_chain(
8003 &self.state.config.agents,
8004 self.state.config.roles.synthesizer.as_ref(),
8005 &agent::installed,
8006 "synthesizer",
8007 ) {
8008 Ok(chain) => chain,
8009 Err(e) => {
8010 tracing::warn!("no agent for the change summary: {e:#}");
8011 self.state.event("pr-summary", "no summary agent available");
8012 return Err("no agent was available to summarize it".to_owned());
8013 }
8014 };
8015 let subjects = git::subjects(&winner.worktree, &base, head)
8016 .await
8017 .unwrap_or_default();
8018 let stat = git::diff_stat(&winner.worktree, &base, head)
8019 .await
8020 .unwrap_or_default();
8021 let stat: String = stat.chars().take(6_000).collect();
8022 let mut prompt = crate::prompt::change_summary_in(
8023 &base,
8024 head,
8025 &subjects,
8026 stat.trim(),
8027 &self.state.config.graph.github_language,
8028 );
8029 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8030 let sessions = self.state.config.graph.sessions;
8031 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8032 let prompts = self.state.config.prompts.clone();
8033 let cache = self.state.config.cache_dir();
8034 let run = self.state.id.clone();
8035 for (i, spec) in chain.into_iter().enumerate() {
8036 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8037 let github_spec = match agent::for_github_text(
8038 &spec,
8039 &self.state.config.graph.github_language,
8040 &winner.worktree,
8041 ) {
8042 Ok(spec) => spec,
8043 Err(e) => {
8044 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8045 continue;
8046 }
8047 };
8048 let job = SeatJob {
8049 spec: github_spec,
8050 seat,
8051 cwd: winner.worktree.clone(),
8052 prompt: prompt.clone(),
8053 timeout,
8054 allow_write: false,
8055 sessions,
8056 artifacts: agent::artifacts_dir(&self.state.dir()),
8057 stem: if i == 0 {
8058 "change-summary".to_owned()
8059 } else {
8060 format!("change-summary-{}", spec.id)
8061 },
8062 handover: None,
8063 };
8064 let ctx = WaveCtx {
8065 run: &run,
8066 node: "pr-summary",
8067 prompts: &prompts,
8068 cache: cache.as_deref(),
8069 round: None,
8070 carry_seats: false,
8071 pauses: Some((&self.pause, &self.interrupt)),
8072 };
8073 let (seat, outcome) =
8074 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8075 self.state.seats.insert(seat.key.clone(), seat);
8076 let text = match outcome {
8077 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8078 AgentOutcome::ResourceBlocked(why) => {
8079 if self.state.resource_block.is_none() {
8082 self.state.resource_block = Some(crate::run::ResourceBlock {
8083 cause: crate::run::ResourceBlockCause::Other,
8084 node: "pr-summary".to_owned(),
8085 kind: "shared resource".to_owned(),
8086 detail: why,
8087 at: Timestamp::now(),
8088 });
8089 }
8090 return Err("a shared resource was busy".to_owned());
8091 }
8092 _ => continue,
8093 };
8094 let blind = &self.state.config.blind;
8095 let Some(title) = summary_title(&text) else {
8096 continue;
8097 };
8098 let body: String = summary_without_title(&text)
8099 .lines()
8100 .filter(|l| !l.contains("magi:run/"))
8101 .collect::<Vec<_>>()
8102 .join("\n");
8103 let body = blind::sanitize_prose(body.trim(), blind);
8104 let title = blind::sanitize_prose(&title, blind);
8105 if body.trim().is_empty() || title.trim().is_empty() {
8106 continue;
8107 }
8108 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8109 continue;
8110 }
8111 if git::rev_parse(&winner.worktree, "HEAD")
8113 .await
8114 .ok()
8115 .as_deref()
8116 != Some(head)
8117 {
8118 return Err("the branch moved while the summary was written".to_owned());
8119 }
8120 if git::merge_base(&winner.worktree, &tracking, head)
8121 .await
8122 .ok()
8123 .as_deref()
8124 != Some(base.as_str())
8125 || crate::pr_outcome::summary_evidence(
8126 &self.state,
8127 crate::pr_outcome::FinalHead::At(head),
8128 ) != evidence
8129 {
8130 return Err("the summary evidence changed while it was written".to_owned());
8131 }
8132 self.state.change_summary = Some(crate::run::ChangeSummary {
8133 head: head.to_owned(),
8134 base,
8135 evidence: fingerprint,
8136 title,
8137 body,
8138 });
8139 self.state.event(
8140 "pr-summary",
8141 format!("change summary written by {} for the final diff", spec.id),
8142 );
8143 if let Err(e) = self.state.save() {
8144 tracing::warn!("could not save the change summary: {e:#}");
8145 }
8146 return Ok(());
8147 }
8148 self.state
8149 .event("pr-summary", "no agent produced a change summary");
8150 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8151 }
8152
8153 async fn rewrite_pr_text(
8163 &mut self,
8164 winner: &Candidate,
8165 pr: &PrMessage,
8166 violations: &[crate::github_text::Violation],
8167 ) -> Option<String> {
8168 let prompt = format!(
8169 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8170 pr.title, pr.body
8171 );
8172 let sessions = self.state.config.graph.sessions;
8173 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8174 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8177 let key = format!("impl-{}", winner.label);
8178 if let Some(spec) = self
8179 .state
8180 .config
8181 .agents
8182 .iter()
8183 .find(|a| a.id == winner.agent)
8184 .cloned()
8185 && let Some(existing) = self.state.seats.get(&key)
8186 && existing.agent == spec.id
8187 && agent::has_session(spec.kind, existing, sessions)
8188 {
8189 attempts.push((spec, Some(existing.clone())));
8190 }
8191 match agent::pick_chain(
8192 &self.state.config.agents,
8193 self.state.config.roles.language_judge.as_ref(),
8194 &agent::installed,
8195 "language_judge",
8196 ) {
8197 Ok(chain) => {
8198 for spec in chain {
8199 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8200 continue;
8201 }
8202 attempts.push((spec, None));
8203 }
8204 }
8205 Err(e) if attempts.is_empty() => {
8206 tracing::warn!("no agent for the description rewrite: {e:#}");
8207 self.state
8208 .event("github-text", "no rewrite agent available");
8209 return None;
8210 }
8211 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8212 }
8213 let prompts = self.state.config.prompts.clone();
8214 let cache = self.state.config.cache_dir();
8215 let run = self.state.id.clone();
8216 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8217 let resumed = resumed_seat.is_some();
8218 let seat = resumed_seat.unwrap_or_else(|| {
8219 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8220 });
8221 let github_spec = match agent::for_github_text(
8222 &spec,
8223 &self.state.config.graph.github_language,
8224 &winner.worktree,
8225 ) {
8226 Ok(spec) => spec,
8227 Err(e) => {
8228 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8229 continue;
8230 }
8231 };
8232 let job = SeatJob {
8233 spec: github_spec,
8234 seat,
8235 cwd: winner.worktree.clone(),
8236 prompt: prompt.clone(),
8237 timeout: retry_budget(timeout_full, resumed),
8238 allow_write: false,
8239 sessions,
8240 artifacts: agent::artifacts_dir(&self.state.dir()),
8241 stem: if i == 0 {
8242 "github-text-rewrite".to_owned()
8243 } else {
8244 format!("github-text-rewrite-{}", spec.id)
8245 },
8246 handover: None,
8247 };
8248 let ctx = WaveCtx {
8249 run: &run,
8250 node: "github-text",
8251 prompts: &prompts,
8252 cache: cache.as_deref(),
8253 round: None,
8254 carry_seats: false,
8255 pauses: Some((&self.pause, &self.interrupt)),
8256 };
8257 let (seat, outcome) =
8258 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8259 self.state.seats.insert(seat.key.clone(), seat);
8260 match outcome {
8261 AgentOutcome::Ok(o)
8262 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8263 {
8264 return Some(o.text);
8265 }
8266 AgentOutcome::ResourceBlocked(_) => return None,
8267 _ => {}
8268 }
8269 }
8270 None
8271 }
8272
8273 fn retire_github_text_question(&mut self, why: &str) {
8275 let Some(id) = self
8276 .state
8277 .github_text
8278 .as_ref()
8279 .and_then(|g| g.question.clone())
8280 else {
8281 return;
8282 };
8283 let _ = ask::Questions::open().update(&id, |q| {
8284 q.abandon(why);
8285 Ok(())
8286 });
8287 }
8288
8289 async fn settle_withheld(
8293 &mut self,
8294 winner: &Candidate,
8295 pr: &PrMessage,
8296 mut g: crate::run::GithubTextAsk,
8297 ) -> Result<Option<PrMessage>> {
8298 use crate::github_text as gt;
8299 let store = ask::Questions::open();
8300 let timeout = self.state.config.graph.answer_timeout;
8301 if !g.resolved {
8302 let mut standing = g
8303 .question
8304 .as_ref()
8305 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8306 if let Some(q) = standing.as_ref()
8307 && q.status.open()
8308 && gt::expired(q, timeout)
8309 {
8310 let why = format!("no answer within {}s of asking", timeout.max(1));
8311 standing = store
8312 .update(&q.id, |q| {
8313 q.abandon(&why);
8314 Ok(())
8315 })
8316 .ok()
8317 .map(|(q, ())| q);
8318 }
8319 let mut ask_again: Option<Option<Vec<String>>> = None;
8320 match standing {
8321 None if g.asks <= 1 => {
8324 g.asks = 0;
8325 ask_again = Some(None);
8326 }
8327 None => {
8328 g.resolved = true;
8329 g.chosen_title = None;
8330 g.choice = Some(GithubTextChoice::Fallback);
8331 }
8332 Some(q) if q.status.open() => {
8333 self.park_on_github_text(&g)?;
8334 return Ok(None);
8335 }
8336 Some(q) => match gt::read_reply(&q) {
8337 gt::Reply::Fallback => {
8338 g.resolved = true;
8339 g.chosen_title = None;
8340 g.choice = Some(GithubTextChoice::Fallback);
8341 }
8342 gt::Reply::Original => {
8343 g.resolved = true;
8344 g.chosen_title = None;
8345 g.choice = Some(GithubTextChoice::Original);
8346 }
8347 gt::Reply::Rewrite => {
8348 g.resolved = true;
8349 g.chosen_title = None;
8350 g.choice = Some(if g.rewrite.is_some() {
8351 GithubTextChoice::Rewrite
8352 } else {
8353 GithubTextChoice::Fallback
8354 });
8355 }
8356 gt::Reply::Title(text) => {
8357 let first = text
8358 .lines()
8359 .map(str::trim)
8360 .find(|l| !l.is_empty())
8361 .unwrap_or("");
8362 let decision = if first.is_empty() {
8363 None
8364 } else {
8365 let candidate = PrMessage {
8366 title: first.to_owned(),
8367 body: String::new(),
8368 };
8369 self.judge_pr_language(&winner.worktree, &candidate).await
8370 };
8371 match gt::vet_title(
8372 &text,
8373 decision,
8374 self.state.config.graph.github_english(),
8375 ) {
8376 Ok(title) => {
8377 g.resolved = true;
8378 g.chosen_title = Some(title);
8379 g.choice = Some(GithubTextChoice::MyTitle);
8380 }
8381 Err(cats) if g.asks < 2 => {
8382 ask_again = Some(Some(
8383 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8384 ))
8385 }
8386 Err(_) => {
8387 g.resolved = true;
8388 g.chosen_title = None;
8389 g.choice = Some(GithubTextChoice::Fallback);
8390 }
8391 }
8392 }
8393 },
8394 }
8395 if let Some(rejected) = ask_again {
8396 let w = gt::Withheld {
8399 title: g.title,
8400 body: g.body,
8401 categories: g.categories.clone(),
8402 };
8403 let artifact = match crate::run::write_artifact(
8406 &self.state,
8407 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8408 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8409 ) {
8410 Ok(p) => Some(p),
8411 Err(e) => {
8412 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8413 None
8414 }
8415 };
8416 let (detail, comparison) = gt::question_content(
8417 &w,
8418 &pr.title,
8419 &pr.body,
8420 &self.state.id,
8421 rejected.as_deref(),
8422 artifact.as_deref(),
8423 g.rewrite.as_ref(),
8424 );
8425 let mut q = ask::Question::new(
8426 self.state.id.clone(),
8427 gt::ASK_NODE.to_owned(),
8428 gt::ASK_SEAT.to_owned(),
8429 gt::question_summary(&self.state.config.graph.language, &w),
8430 detail,
8431 gt::question_choices(&w, g.rewrite.as_ref()),
8432 );
8433 q.posting_comparison = Some(comparison);
8434 q.answer_timeout = timeout;
8435 g.question = Some(q.id.clone());
8438 g.asks += 1;
8439 self.state.github_text = Some(g.clone());
8440 self.state.event(
8441 "github-text",
8442 format!("asking the owner about the withheld text ({})", q.short()),
8443 );
8444 self.state.save()?;
8445 store
8446 .put_executable(&mut q)
8447 .context("file the posting-gate question")?;
8448 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8449 tracing::warn!(
8450 "could not notify about posting-gate question {}: {e:#}",
8451 q.short()
8452 );
8453 }
8454 self.park_on_github_text(&g)?;
8455 return Ok(None);
8456 }
8457 }
8458 let resolved = g.chosen_title.clone();
8459 let original = g.choice == Some(GithubTextChoice::Original);
8460 self.state.event(
8462 "github-text",
8463 if original {
8464 format!(
8465 "posting the original text for: {}",
8466 match (g.title, g.body) {
8467 (true, true) => "title, description",
8468 (true, false) => "title",
8469 _ => "description",
8470 }
8471 )
8472 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8473 "posting the saved rewrite title and description".to_owned()
8474 } else if resolved.is_some() {
8475 "posting the owner's replacement title".to_owned()
8476 } else {
8477 "posting the neutral text for the withheld fields".to_owned()
8478 },
8479 );
8480 self.state.github_text = Some(g.clone());
8481 self.state.save()?;
8482 if g.choice == Some(GithubTextChoice::Rewrite)
8483 && let Some(r) = g.rewrite.as_ref()
8484 {
8485 return Ok(Some(PrMessage {
8486 title: r.title.clone(),
8487 body: r.body.clone(),
8488 }));
8489 }
8490 let w = gt::Withheld {
8491 title: g.title,
8492 body: g.body,
8493 categories: g.categories.clone(),
8494 };
8495 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8496 if let Some(t) = resolved {
8497 title = t;
8498 }
8499 let english = gt::LanguageDecision {
8501 title_english: true,
8502 body_english: true,
8503 body_complete: true,
8504 };
8505 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8506 if original {
8507 if g.title {
8510 title = pr.title.clone();
8511 }
8512 if g.body {
8513 body = pr.body.clone();
8514 }
8515 }
8516 Ok(Some(PrMessage { title, body }))
8517 }
8518
8519 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8520 self.state.github_text = Some(g.clone());
8521 self.state.end_pass_parked(ParkCause::Question);
8522 self.state.parked = true;
8523 self.state.event(
8524 "github-text",
8525 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8526 );
8527 self.state.save()
8528 }
8529
8530 async fn judge_pr_language(
8533 &mut self,
8534 cwd: &Path,
8535 pr: &PrMessage,
8536 ) -> Option<crate::github_text::LanguageDecision> {
8537 self.state.config.roles.language_judge.as_ref()?;
8538 if !self.state.config.graph.language_gate() {
8539 return None;
8540 }
8541 let decision =
8542 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8543 self.state.event(
8544 "github-text",
8545 if decision.is_some() {
8546 "language decided by the language judge"
8547 } else {
8548 "language judge unavailable; using built-in heuristics"
8549 },
8550 );
8551 decision
8552 }
8553
8554 async fn merge(&mut self) -> Result<()> {
8555 if self.resource_halt()? || self.hold_base_sync().await? {
8556 return Ok(());
8557 }
8558 if self
8573 .state
8574 .base_sync
8575 .as_ref()
8576 .is_some_and(|s| s.conflict.is_some())
8577 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8578 != Some(RunStatus::Gating)
8579 || !self.state.gate_allows_progress()
8587 {
8588 return Ok(());
8589 }
8590 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8599 return Ok(());
8600 }
8601 let Some(winner) = self.state.winner().cloned() else {
8602 return Ok(());
8603 };
8604 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8605 if self.state.gate_head.as_deref() != Some(head.as_str()) {
8606 self.state.status = RunStatus::Blocked;
8607 self.state.event(
8608 "gate",
8609 "winner changed after gate; fresh verification required",
8610 );
8611 self.state.save()?;
8612 return Ok(());
8613 }
8614 let wait = self.repository_wait();
8615 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8616 return Ok(());
8617 }
8618 let repo = self.state.repo.clone();
8619 let base = self.state.base_branch.clone();
8620 let mode = self.state.config.merge.mode;
8621 let style = self.state.config.merge.style;
8622 let facts = if is_review_run(&self.state) {
8623 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8624 let start = review_base(
8625 &repo,
8626 &self.state.config.merge.remote,
8627 &base,
8628 &self.state.base_commit,
8629 &winner.branch,
8630 )
8631 .await;
8632 branch_facts(&repo, &start, &winner.branch).await
8633 } else {
8634 None
8635 };
8636 let leaked = if is_review_run(&self.state) {
8639 leaked_subjects(&self.state, &winner.branch).await
8640 } else {
8641 Some(Vec::new())
8642 };
8643 let Some(pr) = self
8647 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8648 .await?
8649 else {
8650 return Ok(());
8651 };
8652 let message = pr.commit_message();
8653
8654 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
8655 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
8656 return Ok(());
8657 }
8658 let outcome = match mode {
8659 MergeMode::None => MergeOutcome {
8660 mode,
8661 ok: true,
8662 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8663 empty: false,
8664 },
8665 MergeMode::Pr | MergeMode::Local
8666 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8667 {
8668 MergeOutcome {
8669 mode,
8670 ok: false,
8671 detail: empty_candidate_detail(&self.state, &base),
8672 empty: true,
8673 }
8674 }
8675 MergeMode::Local => {
8676 let on = git::current_branch(&repo).await?;
8677 if on.as_deref() != Some(base.as_str()) {
8678 MergeOutcome {
8679 mode,
8680 ok: false,
8681 detail: format!(
8682 "{} has {} checked out, not the base branch {base}",
8683 repo.display(),
8684 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8685 ),
8686 empty: false,
8687 }
8688 } else if !git::is_clean(&repo).await? {
8689 MergeOutcome {
8690 mode,
8691 ok: false,
8692 detail: format!("{} is dirty; refusing to merge", repo.display()),
8693 empty: false,
8694 }
8695 } else {
8696 let out = match style {
8697 MergeStyle::Merge => {
8698 git::merge_no_ff(&repo, &winner.branch, &message).await?
8699 }
8700 MergeStyle::Squash => {
8701 git::merge_squash(&repo, &winner.branch, &message).await?
8702 }
8703 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8704 };
8705 MergeOutcome {
8706 mode,
8707 ok: out.ok(),
8708 detail: if out.ok() { out.stdout } else { out.stderr },
8709 empty: false,
8710 }
8711 }
8712 }
8713 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8714 self.state.pr_publication_pending = true;
8715 self.state.save()?;
8716 let out =
8717 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8718 self.state.pr_publication_pending = out.is_err();
8719 MergeOutcome {
8720 mode,
8721 ok: out.is_ok(),
8722 detail: match out {
8723 Ok(url) => url,
8724 Err(e) => format!("{e:#}"),
8725 },
8726 empty: false,
8727 }
8728 }
8729 MergeMode::Pr => {
8730 let remote = self.state.config.merge.remote.clone();
8731 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8732 let pushed = if self.state.foreign_checkout.is_some() {
8733 let refspec = format!("{delivery_head}:refs/heads/{public}");
8734 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
8735 } else {
8736 git::push(&winner.worktree, &remote, &winner.branch).await?
8737 };
8738 if !pushed.ok() {
8739 MergeOutcome {
8740 mode,
8741 ok: false,
8742 detail: pushed.stderr,
8743 empty: false,
8744 }
8745 } else {
8746 if self.state.foreign_checkout.is_some() {
8747 self.state.pr_pushed_head = Some(delivery_head.clone());
8748 }
8749 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
8755 let out = match pr_merge_plan(found) {
8756 PrPlan::Create => {
8757 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
8758 .await
8759 }
8760 PrPlan::Adopt { url, title } => {
8761 self.state
8762 .event("merge", format!("Pr: adopted open pull request {url}"));
8763 if self.state.github_text.as_ref().is_some_and(|g| {
8764 g.resolved
8765 && g.choice == Some(GithubTextChoice::Rewrite)
8766 && g.rewrite.is_some()
8767 }) {
8768 land::set_pr_text_verbatim(
8771 &winner.worktree,
8772 &url,
8773 &pr.title,
8774 &pr.body,
8775 )
8776 .await
8777 .map(|()| url)
8778 } else {
8779 if title != pr.title
8780 && (!is_review_run(&self.state)
8781 || leaked
8782 .as_deref()
8783 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8784 && let Err(e) =
8785 if self.state.github_text.as_ref().is_some_and(|g| {
8786 g.resolved
8787 && g.title
8788 && g.choice == Some(GithubTextChoice::Original)
8789 }) {
8790 land::set_pr_title_verbatim(
8792 &winner.worktree,
8793 &url,
8794 &pr.title,
8795 )
8796 .await
8797 } else {
8798 land::set_pr_title(
8799 &mut self.state,
8800 &winner.worktree,
8801 &url,
8802 &pr.title,
8803 )
8804 .await
8805 }
8806 {
8807 tracing::warn!("could not refresh title of {url}: {e:#}");
8808 self.state
8809 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8810 }
8811 Ok(url)
8812 }
8813 }
8814 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8815 };
8816 match out {
8817 Ok(url) => MergeOutcome {
8818 mode,
8819 ok: true,
8820 detail: url,
8821 empty: false,
8822 },
8823 Err(e) => MergeOutcome {
8824 mode,
8825 ok: false,
8826 detail: e.to_string(),
8827 empty: false,
8828 },
8829 }
8830 }
8831 }
8832 };
8833
8834 self.state.status = match (mode, outcome.ok) {
8835 (MergeMode::None, _) => RunStatus::Ready,
8836 (_, true) if outcome.empty => RunStatus::Ready,
8837 (MergeMode::Pr, true) => RunStatus::Ready,
8838 (_, true) => RunStatus::Merged,
8839 (_, false) => RunStatus::Blocked,
8840 };
8841 self.state.event(
8842 "merge",
8843 format!(
8844 "{:?}: {}",
8845 mode,
8846 outcome.detail.lines().next().unwrap_or("")
8847 ),
8848 );
8849 self.state.merge = Some(outcome);
8850 self.state.save()?;
8851
8852 if self.state.config.graph.land
8858 && mode == MergeMode::Pr
8859 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8860 {
8861 self.run_land().await?;
8862 }
8863 self.settle_questions();
8868 Ok(())
8869 }
8870
8871 async fn run_land(&mut self) -> Result<()> {
8882 crate::pr_head::ensure_owner(&self.state)?;
8883 let url = self
8884 .state
8885 .pr
8886 .as_ref()
8887 .map(|pr| pr.url.clone())
8888 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8889 .unwrap_or_default();
8890 let url = url.lines().next().unwrap_or("").trim().to_owned();
8891 if !url.starts_with("http") {
8892 return Ok(());
8893 }
8894 loop {
8897 match land::land(&mut self.state, &url).await {
8898 Ok(_)
8899 if self
8900 .state
8901 .merge_repairs
8902 .last()
8903 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8904 {
8905 if self.repair_merge_review().await? {
8906 continue;
8907 }
8908 }
8909 Ok(pr) if self.state.parked => {
8910 let _ = pr;
8914 }
8915 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8916 self.state.status = RunStatus::Superseded;
8917 self.state.save()?;
8918 }
8919 Ok(pr) => {
8920 self.state.status = match pr.state {
8921 land::PrLifecycle::Merged => RunStatus::Merged,
8922 _ => RunStatus::Blocked,
8923 };
8924 if bump::should_release_bump(self.state.status)
8931 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8932 {
8933 self.state
8939 .event("bump", format!("release bump skipped: {e:#}"));
8940 }
8941 if self.state.status == RunStatus::Merged {
8944 crate::followup::after_merge(&mut self.state, &pr.url).await;
8945 }
8946 self.state.save()?;
8947 }
8948 Err(e) => {
8949 if e.is::<crate::repo_lock::ResourceBlocked>() {
8950 return Err(e);
8951 }
8952 self.state.status = RunStatus::Blocked;
8953 self.state.event("land", format!("gave up: {e}"));
8954 self.state.save()?;
8955 }
8956 }
8957 break;
8958 }
8959 Ok(())
8960 }
8961
8962 async fn repair_merge_review(&mut self) -> Result<bool> {
8965 use crate::run::MergeRepairPhase as Phase;
8966 crate::pr_head::ensure_owner(&self.state)?;
8967 let i = self.state.merge_repairs.len() - 1;
8968 let request = self.state.merge_repairs[i].clone();
8969 if request.phase == Phase::Fixing {
8970 let restored = match self.state.winner().cloned() {
8971 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8972 None => Err(anyhow::anyhow!("repair has no winning branch")),
8973 };
8974 let why = match restored {
8975 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8976 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8977 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8978 };
8979 self.hold_merge_repair(i, why)?;
8980 return Ok(false);
8981 }
8982 if request.phase == Phase::Reviewing && self.state.gate_ran {
8987 let winner = self
8988 .state
8989 .winner()
8990 .context("repair has no winning branch")?;
8991 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8992 if self.state.gate_head.as_deref() != Some(head.as_str())
8993 || !git::is_clean(&winner.worktree).await?
8994 {
8995 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8996 return Ok(false);
8997 }
8998 }
8999 let failure = if i >= self.state.config.graph.land_rounds
9000 || self.state.config.graph.review_rounds == 0
9001 {
9002 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9003 } else {
9004 None
9005 };
9006 if let Some(why) = failure {
9007 self.hold_merge_repair(i, why)?;
9008 return Ok(false);
9009 }
9010 if request.phase == Phase::Requested {
9011 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9012 Ok(pr) => pr,
9013 Err(e) => {
9014 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9015 return Ok(false);
9016 }
9017 };
9018 self.state.merge_repairs[i].phase = Phase::Fixing;
9019 self.state.land_approval = None;
9020 self.state.land_armed_head = None;
9021 self.state.save()?;
9022 let evidence = serde_json::to_string_pretty(&request)?;
9025 let reason = format!(
9026 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9027 );
9028 let budget = self.state.config.graph.land_rounds;
9029 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9030 match result {
9031 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9032 Ok(land::Fixed::Failed(why)) => {
9033 self.hold_merge_repair(
9034 i,
9035 format!(
9036 "repair failed: {why}; inspect the branch and run an explicit review"
9037 ),
9038 )?;
9039 return Ok(false);
9040 }
9041 Err(e) => {
9042 self.hold_merge_repair(
9043 i,
9044 format!(
9045 "repair failed: {e:#}; inspect the branch and run an explicit review"
9046 ),
9047 )?;
9048 return Ok(false);
9049 }
9050 }
9051 self.begin_merge_repair_review(i)?;
9055 }
9056 self.review_loop().await?;
9057 self.gate().await?;
9058 if self.resource_halt()? || self.state.parked {
9059 return Ok(false);
9060 }
9061 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9062 != Some(RunStatus::Gating)
9063 || !self.state.gate_allows_progress()
9064 {
9065 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9066 return Ok(false);
9067 }
9068 let winner = self
9071 .state
9072 .winner()
9073 .cloned()
9074 .context("repair has no winning branch")?;
9075 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9076 if self.state.gate_head.as_deref() != Some(after.as_str())
9077 || !git::is_clean(&winner.worktree).await?
9078 {
9079 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9080 return Ok(false);
9081 }
9082 let expected = self
9083 .state
9084 .pr_pushed_head
9085 .as_deref()
9086 .unwrap_or(&request.head)
9087 .to_owned();
9088 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9089 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9092 return Ok(false);
9093 }
9094 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9095 let refspec = format!("{after}:refs/heads/{public}");
9096 let push = git::git_raw(
9097 &winner.worktree,
9098 &["push", &lease, &self.state.config.merge.remote, &refspec],
9099 )
9100 .await;
9101 match push {
9102 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9103 Ok(out) => {
9104 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9105 return Ok(false);
9106 }
9107 Err(e) => {
9108 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9109 return Ok(false);
9110 }
9111 }
9112 self.state.merge_repairs[i].phase = Phase::Complete;
9113 self.state.status = RunStatus::Landing;
9114 self.state.event(
9115 "repair",
9116 "new review and gate completed; fresh owner merge decision required",
9117 );
9118 self.state.save()?;
9119 Ok(true)
9120 }
9121
9122 fn merge_repair_artifacts(&self) -> PathBuf {
9123 let artifacts = agent::artifacts_dir(&self.state.dir());
9124 match self
9125 .state
9126 .merge_repairs
9127 .last()
9128 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9129 {
9130 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9131 None => artifacts,
9132 }
9133 }
9134
9135 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9136 if let Some(e2e) = self.state.head_e2e.take() {
9137 self.state
9138 .verification_history
9139 .push(crate::run::VerificationEvidence {
9140 superseded_by: None,
9141 archived_at: Some(Timestamp::now()),
9142 fixes_spent: Some(self.state.gate_fixes.len()),
9143 gate_head: None,
9144 gate_ran: false,
9145 gate: Vec::new(),
9146 e2e: Some(e2e),
9147 });
9148 }
9149 self.state.reviews.clear();
9150 self.state.gate.clear();
9151 self.state.gate_ran = false;
9152 self.state.gate_head = None;
9153 self.state.gate_fixes.clear();
9154 self.state.gate_repair_e2e = None;
9155 self.state.pre_gate.clear();
9156 self.state.pre_gate_commit = None;
9157 self.state.contested_handoff = None;
9158 self.state.land_reruns = None;
9159 self.state.land_approval = None;
9160 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9161 self.state.status = RunStatus::Reviewing;
9162 self.state.save()
9163 }
9164
9165 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9166 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9167 self.state.merge_repairs[index].failure = Some(why.clone());
9168 self.state.status = RunStatus::Blocked;
9169 self.state.event("repair", why);
9170 self.state.save()
9171 }
9172
9173 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9177 if let Some(existing) = self.state.seats.get(key)
9178 && existing.agent == agent
9179 {
9180 return existing.clone();
9181 }
9182 let fresh = if self.state.seats.contains_key(key) {
9185 handover_seat(key, agent, self.state.next_seat_seed())
9186 } else {
9187 SeatState::new(key, agent, self.state.seed)
9188 };
9189 self.state.seats.insert(key.to_owned(), fresh.clone());
9190 fresh
9191 }
9192
9193 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9199 match self.state.seats.get(key) {
9200 Some(s) if s.agent != spec.id => {
9201 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9202 }
9203 _ => spec,
9204 }
9205 }
9206
9207 fn view(&self, c: &Candidate) -> CandidateView {
9209 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9210 .unwrap_or_default();
9211 let (patch, _) = blind::sanitize_patch(
9212 &format!("candidate {} patch", c.label),
9213 &raw,
9214 &self.state.config.blind,
9215 );
9216 CandidateView {
9217 label: c.label,
9218 branch: c.branch.clone(),
9219 summary: c.summary.clone(),
9220 stat: c.stat.clone(),
9221 patch,
9222 }
9223 }
9224
9225 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9227 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9228 prompt::judge(
9229 "(see above)",
9230 &views,
9231 self.roles.judges.len(),
9232 base_short,
9233 "en",
9234 )
9235 }
9236
9237 fn vote_prompt_full(
9243 &self,
9244 j: usize,
9245 labels: &[char],
9246 language: &str,
9247 candidates: &[Candidate],
9248 base_short: &str,
9249 ) -> String {
9250 let mut text = format!(
9251 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9252 prompt::final_vote(labels, language),
9253 self.state.instruction,
9254 self.candidate_block(candidates, base_short)
9255 );
9256 if let Some(own) = self
9257 .state
9258 .judgements
9259 .get(j)
9260 .filter(|r| !r.ranking.is_empty())
9261 {
9262 let reasons = own
9263 .reasons
9264 .iter()
9265 .map(|(k, v)| format!("- {k}: {v}"))
9266 .collect::<Vec<_>>()
9267 .join("\n");
9268 text.push_str(&format!(
9269 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9270 own.ranking.iter().collect::<String>(),
9271 if reasons.is_empty() {
9272 ""
9273 } else {
9274 ", because:\n"
9275 }
9276 ));
9277 }
9278 if !self.state.deliberation.is_empty() {
9279 text.push_str("\n# What was argued before this vote\n");
9280 for t in self.transcript(&[], j) {
9281 text.push_str(&format!(
9282 "\n## {}{}\n\n{}\n",
9283 t.who,
9284 if t.is_self { " (you)" } else { "" },
9285 t.body.trim()
9286 ));
9287 }
9288 }
9289 text
9290 }
9291
9292 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9299 let mut turns = Vec::new();
9300 for j in &self.state.judgements {
9301 if j.ranking.is_empty() {
9302 continue;
9303 }
9304 let reasons = j
9305 .reasons
9306 .iter()
9307 .map(|(k, v)| format!("- {k}: {v}"))
9308 .collect::<Vec<_>>()
9309 .join("\n");
9310 turns.push(Turn {
9311 who: format!("Judge {} (opening ranking)", j.judge),
9312 is_self: j.judge == self_idx + 1,
9313 body: format!(
9314 "Ranked {}{}{reasons}",
9315 j.ranking.iter().collect::<String>(),
9316 if reasons.is_empty() {
9317 ""
9318 } else {
9319 ", because:\n"
9320 }
9321 ),
9322 });
9323 }
9324 for t in self
9325 .state
9326 .deliberation
9327 .iter()
9328 .flat_map(|r| r.turns.iter())
9329 .chain(current)
9330 {
9331 turns.push(Turn {
9332 who: format!("Judge {}", t.judge),
9333 is_self: t.judge == self_idx + 1,
9334 body: t.body.clone(),
9335 });
9336 }
9337 turns
9338 }
9339}
9340
9341fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9343 agent::has_session(spec.kind, seat, sessions)
9344}
9345
9346fn next_untried_in_roster<'a>(
9367 roster: &'a [AgentSpec],
9368 start: usize,
9369 tried: &BTreeSet<String>,
9370) -> Option<&'a AgentSpec> {
9371 roster
9372 .get(start + 1..)?
9373 .iter()
9374 .find(|s| !tried.contains(&s.id))
9375}
9376
9377fn pick_successor<'a>(
9398 roster: &'a [AgentSpec],
9399 start: usize,
9400 tried: &BTreeSet<String>,
9401 carried: Option<&BTreeSet<String>>,
9402 others: &BTreeSet<String>,
9403) -> Option<&'a AgentSpec> {
9404 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9405 roster
9406 .get(start + 1..)
9407 .and_then(|tail| {
9408 tail.iter()
9409 .filter(free)
9410 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9411 })
9412 .or_else(|| {
9413 carried
9414 .is_some()
9415 .then(|| roster.iter().find(free))
9416 .flatten()
9417 })
9418 .or_else(|| match carried {
9419 Some(c) => next_for_seat(roster, start, tried, c),
9420 None => next_untried_in_roster(roster, start, tried),
9421 })
9422}
9423
9424fn next_for_seat<'a>(
9434 roster: &'a [AgentSpec],
9435 start: usize,
9436 round_tried: &BTreeSet<String>,
9437 carried_failed: &BTreeSet<String>,
9438) -> Option<&'a AgentSpec> {
9439 roster
9440 .get(start + 1..)?
9441 .iter()
9442 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9443 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9444}
9445
9446fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9452 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9453 return spec;
9454 };
9455 let ok = |id: &str| !h.failed.contains(id);
9456 if let Some(last) = h.last_ok.as_deref()
9457 && ok(last)
9458 && let Some(s) = roster.iter().find(|s| s.id == last)
9459 {
9460 return s.clone();
9461 }
9462 if ok(&spec.id) {
9463 return spec;
9464 }
9465 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9466 roster
9467 .get(from + 1..)
9468 .into_iter()
9469 .flatten()
9470 .chain(roster.iter())
9471 .find(|s| ok(&s.id))
9472 .cloned()
9473 .unwrap_or(spec)
9474}
9475
9476pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9480 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9481}
9482
9483const TIMED_OUT: &str = "timed out";
9486
9487impl FailClass {
9488 fn of(out: &AgentOutcome) -> Option<Self> {
9490 match out {
9491 AgentOutcome::Ok(_) => None,
9492 AgentOutcome::Quota(_) => Some(Self::Quota),
9493 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9494 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9495 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9496 AgentOutcome::ResourceBlocked(_) => None,
9499 }
9500 }
9501
9502 fn stem_word(&self) -> &'static str {
9504 match self {
9505 Self::Quota => "quota",
9506 _ => "handover",
9507 }
9508 }
9509}
9510
9511fn failure_signature(msg: &str) -> String {
9515 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9516 if let Some(code) = line
9519 .strip_prefix("process exited with no usable output (exit code ")
9520 .and_then(|s| s.strip_suffix(')'))
9521 .and_then(|s| s.parse::<i32>().ok())
9522 {
9523 line = format!("exited with some({code}) and no usable output");
9524 } else if matches!(
9525 line.as_str(),
9526 "process exited with no usable output (no exit code reported)"
9527 | "process exited with no usable output (exit code unavailable)"
9528 ) {
9529 line = "exited with none and no usable output".to_owned();
9530 }
9531 let mut out = Vec::new();
9532 for word in line.split_whitespace() {
9533 if word.contains('/') || word.contains('\\') {
9534 out.push("<path>".to_owned());
9535 continue;
9536 }
9537 let mut w = String::new();
9538 let mut in_digits = false;
9539 for c in word.chars() {
9540 if c.is_ascii_digit() {
9541 if !in_digits {
9542 w.push('#');
9543 }
9544 in_digits = true;
9545 } else {
9546 in_digits = false;
9547 w.push(c);
9548 }
9549 }
9550 out.push(w);
9551 }
9552 out.join(" ").chars().take(120).collect()
9553}
9554
9555fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9562 match cur {
9563 FailClass::Quota | FailClass::Timeout => true,
9564 FailClass::Other(_) => prev != Some(cur),
9565 }
9566}
9567
9568fn fail_reason(out: &AgentOutcome) -> String {
9570 match out {
9571 AgentOutcome::Ok(_) => String::new(),
9572 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9573 AgentOutcome::Dropped(o) => format!(
9574 "the CLI dropped the stream ({})",
9575 o.dropped
9576 .as_ref()
9577 .map(|d| d.why.as_str())
9578 .unwrap_or("it ended without delivering its answer")
9579 ),
9580 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9581 e.lines().next().unwrap_or("").chars().take(160).collect()
9582 }
9583 }
9584}
9585
9586pub(crate) fn record_handover(
9589 state: &mut RunState,
9590 node: &str,
9591 seat: &str,
9592 from: &str,
9593 to: &str,
9594 class: &FailClass,
9595 reason: &str,
9596) {
9597 let message = if *class == FailClass::Quota {
9598 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9599 } else {
9600 format!("{seat}: handed over {from} -> {to} ({reason})")
9601 };
9602 state.event(node, message);
9603 state.handovers.push(Handover {
9604 at: Timestamp::now(),
9605 node: node.to_owned(),
9606 seat: seat.to_owned(),
9607 from: from.to_owned(),
9608 to: to.to_owned(),
9609 reason: reason.to_owned(),
9610 });
9611}
9612
9613fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9627 commands.iter().any(|c| c.exit_code.is_none())
9628}
9629
9630fn verified_noop_claim(
9643 usable: bool,
9644 commands: &[agent::CommandEvidence],
9645 text: &str,
9646) -> Option<String> {
9647 (usable && !has_unconfirmed_command(commands))
9648 .then(|| verdict::verified_noop(text))
9649 .flatten()
9650}
9651
9652fn short(commit: &str) -> String {
9653 commit.chars().take(7).collect()
9654}
9655
9656fn make_executable(path: &Path) -> Result<()> {
9657 #[cfg(unix)]
9658 {
9659 use std::os::unix::fs::PermissionsExt as _;
9660 let mut perms = std::fs::metadata(path)?.permissions();
9661 perms.set_mode(0o755);
9662 std::fs::set_permissions(path, perms)?;
9663 }
9664 #[cfg(not(unix))]
9665 {
9666 let _ = path;
9667 }
9668 Ok(())
9669}
9670
9671struct WaveCtx<'a> {
9678 run: &'a str,
9681 node: &'a str,
9683 prompts: &'a Prompts,
9684 cache: Option<&'a Path>,
9686 round: Option<usize>,
9689 carry_seats: bool,
9693 pauses: Option<(&'a Pause, &'a Pause)>,
9696}
9697
9698fn repository_wait(
9700 state: &RunState,
9701 pauses: Option<(&Pause, &Pause)>,
9702) -> crate::repo_lock::WaitPolicy {
9703 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9704 let inherited = crate::repo_lock::WaitPolicy::current();
9705 crate::repo_lock::WaitPolicy::new(
9706 Duration::from_secs(state.config.graph.timeout_resource_wait),
9707 move || {
9708 inherited.cancelled()
9709 || pauses
9710 .as_ref()
9711 .is_some_and(|(a, b)| a.parked() || b.parked())
9712 },
9713 )
9714}
9715
9716fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9717 if error.is::<crate::repo_lock::ResourceBlocked>() {
9718 state.resource_block = Some(crate::run::ResourceBlock {
9719 cause: crate::run::ResourceBlockCause::Other,
9720 node: "repository".into(),
9721 kind: if error.to_string().contains("disk pressure") {
9722 "disk pressure"
9723 } else {
9724 "repository coordination"
9725 }
9726 .into(),
9727 detail: error.to_string(),
9728 at: Timestamp::now(),
9729 });
9730 state.event("resource", error.to_string());
9731 }
9732}
9733
9734async fn coordinate_operation(
9735 state: &mut RunState,
9736 cwd: &Path,
9737 wait: &crate::repo_lock::WaitPolicy,
9738) -> Result<Option<crate::repo_lock::SeatLease>> {
9739 let result =
9740 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9741 .await;
9742 if let Err(e) = &result {
9743 record_repository_block(state, e);
9744 }
9745 result
9746}
9747
9748async fn run_one(
9750 job: SeatJob,
9751 sem: Arc<Semaphore>,
9752 ctx: &WaveCtx<'_>,
9753 state: &mut RunState,
9754 attempt: usize,
9755) -> (SeatState, AgentOutcome) {
9756 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9757 .await
9758 .pop()
9759 .expect("one job in, one result out");
9760 (seat, out)
9761}
9762
9763async fn wave(
9769 jobs: Vec<SeatJob>,
9770 sem: Arc<Semaphore>,
9771 ctx: &WaveCtx<'_>,
9772 state: &mut RunState,
9773 attempt: usize,
9774) -> Vec<(usize, SeatState, AgentOutcome)> {
9775 let WaveCtx {
9776 run,
9777 node,
9778 prompts,
9779 cache,
9780 round,
9781 carry_seats: _,
9782 pauses,
9783 } = *ctx;
9784 for job in &jobs {
9785 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9786 }
9787 if let Err(e) = state.save() {
9788 tracing::warn!("could not persist in-progress seats: {e:#}");
9793 }
9794 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9810 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9814 let cache_guard = if let Some(cache_dir) = cache {
9815 if jobs_had_a_writer {
9816 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9817 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9818 match acquire_cache_lease(
9819 state,
9820 cache_dir,
9821 &owner,
9822 resource_limit,
9823 node,
9824 node,
9825 &cancel,
9826 )
9827 .await
9828 {
9829 Ok(Some(g)) => Some(g),
9830 Ok(None) => {
9833 if let Some(block) = &mut state.resource_block {
9834 block.cause = crate::run::ResourceBlockCause::Cancelled {
9835 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9836 };
9837 }
9838 return block_wave(
9839 jobs,
9840 state,
9841 node,
9842 attempt,
9843 "parked while waiting for the shared build cache",
9844 );
9845 }
9846 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9849 Err(e) => {
9850 let why = format!("{e:#}");
9854 state.event(
9855 "resource",
9856 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9857 );
9858 return block_wave(jobs, state, node, attempt, &why);
9859 }
9860 }
9861 } else {
9862 None
9863 }
9864 } else {
9865 None
9866 };
9867 state.seats_spawning(node, attempt);
9870 if let Err(e) = state.save() {
9871 tracing::warn!("could not persist the start of a wave: {e:#}");
9872 }
9873 let slot_limit = resource_limit
9877 + jobs
9878 .iter()
9879 .map(|j| j.timeout)
9880 .max()
9881 .unwrap_or_default()
9882 .saturating_mul(jobs.len() as u32);
9883 let mut set = tokio::task::JoinSet::new();
9884 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9887 let overlay = prompts.overlay(node);
9888 let repository_wait = repository_wait(state, pauses);
9889 for (i, mut job) in jobs.into_iter().enumerate() {
9890 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9891 if cache.is_some() {
9892 job.prompt.push('\n');
9893 job.prompt
9894 .push_str(&prompt::build_cache_note(node, job.allow_write));
9895 }
9896 let sem = Arc::clone(&sem);
9897 let run = run.to_owned();
9898 let node = node.to_owned();
9899 let attachments = if node == "implement" {
9902 state.attachments.clone()
9903 } else {
9904 Vec::new()
9905 };
9906 let cache = cache
9917 .filter(|_| job.allow_write && cache_guard.is_some())
9918 .map(Path::to_path_buf);
9919 let spawned_tx = spawned_tx.clone();
9920 let repository_wait = repository_wait.clone();
9921 let coordination = state.config.coordination.clone();
9922 let fetched_base = state.fetched_base.clone();
9923 set.spawn(async move {
9924 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9928 Ok(p) => p,
9929 Err(_) => {
9930 return (
9931 i,
9932 job.seat,
9933 AgentOutcome::ResourceBlocked(format!(
9934 "no parallel slot became free within {}s",
9935 slot_limit.as_secs()
9936 )),
9937 false,
9938 );
9939 }
9940 };
9941 spawned_tx.send(job.seat.key.clone()).ok();
9942 let mut seat = job.seat;
9943 let out = repository_wait
9944 .scope(agent::invoke(
9945 &job.spec,
9946 &mut seat,
9947 &Invocation {
9948 coordination: coordination.as_ref(),
9949 fetched_base: fetched_base.as_ref(),
9950 cwd: &job.cwd,
9951 prompt: &job.prompt,
9952 timeout: job.timeout,
9953 allow_write: job.allow_write,
9954 sessions: job.sessions,
9955 artifacts: &job.artifacts,
9956 stem: &job.stem,
9957 run: &run,
9958 node: &node,
9959 cache_dir: cache.as_deref(),
9960 attachments: &attachments,
9961 writable: &[],
9962 },
9963 ))
9964 .await;
9965 let interrupted = out
9966 .as_ref()
9967 .err()
9968 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9969 let out = match out {
9970 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9971 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9972 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9980 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9981 Ok(o) => AgentOutcome::Failed(format!(
9982 "process exited with no usable output ({})",
9983 crate::notices::exit_status(o.exit_code)
9984 )),
9985 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9986 AgentOutcome::ResourceBlocked(e.to_string())
9987 }
9988 Err(e) => AgentOutcome::Failed(e.to_string()),
9989 };
9990 (i, seat, out, interrupted)
9991 });
9992 }
9993 drop(spawned_tx);
9994 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9995 loop {
9996 let joined = tokio::select! {
9997 Some(key) = spawned_rx.recv() => {
9998 state.seat_spawned(&key);
10000 if let Err(e) = state.save() {
10001 tracing::warn!("could not persist a seat's start: {e:#}");
10002 }
10003 continue;
10004 }
10005 joined = set.join_next() => joined,
10006 };
10007 let Some(joined) = joined else {
10008 break;
10009 };
10010 let (i, seat, out, interrupted) = match joined {
10011 Ok(v) => v,
10012 Err(e) => {
10016 tracing::error!("agent task panicked: {e}");
10017 continue;
10018 }
10019 };
10020 state.seat_finished(&seat.key);
10021 if interrupted
10022 && node == "implement"
10023 && let Some(label) = state
10024 .candidates
10025 .iter()
10026 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10027 .map(|candidate| candidate.label)
10028 {
10029 state.candidate_preservation.insert(
10030 label,
10031 "interrupted implementation awaiting persistence".into(),
10032 );
10033 state.interrupted_implementations.insert(label);
10034 }
10035 if let AgentOutcome::ResourceBlocked(why) = &out
10036 && state.resource_block.is_none()
10037 {
10038 state.resource_block = Some(crate::run::ResourceBlock {
10039 cause: crate::run::ResourceBlockCause::Other,
10040 node: node.to_owned(),
10041 kind: if why.contains("disk pressure") {
10042 "disk pressure"
10043 } else if why.contains("repository coordination") {
10044 "repository coordination"
10045 } else {
10046 "parallel slot"
10047 }
10048 .to_owned(),
10049 detail: why.clone(),
10050 at: Timestamp::now(),
10051 });
10052 let activity = if interrupted {
10053 "agent execution interrupted"
10054 } else {
10055 "no agent was started"
10056 };
10057 state.event("resource", format!("{node}: {activity} - {why}"));
10058 }
10059 record_jobs(state, node, round, &seat.key, &out);
10060 if let Err(e) = state.save() {
10061 tracing::warn!("could not persist a seat's completion: {e:#}");
10062 }
10063 if collected.len() <= i {
10064 collected.resize_with(i + 1, || None);
10065 }
10066 collected[i] = Some((i, seat, out));
10067 }
10068 if state
10074 .active
10075 .values()
10076 .any(|a| a.node == node && a.attempt == attempt)
10077 {
10078 state
10079 .active
10080 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10081 if let Err(e) = state.save() {
10082 tracing::warn!("could not persist the end of a wave: {e:#}");
10083 }
10084 }
10085 if let Some(cache_dir) = cache
10092 && jobs_had_a_writer
10093 {
10094 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10095 }
10096 if let Some(guard) = cache_guard {
10097 guard.release();
10098 }
10099 collected.into_iter().flatten().collect()
10100}
10101
10102fn record_jobs(
10113 state: &mut RunState,
10114 node: &str,
10115 round: Option<usize>,
10116 seat: &str,
10117 out: &AgentOutcome,
10118) {
10119 let commands: &[agent::CommandEvidence] = match out {
10120 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10121 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10122 };
10123 let checked_at = Timestamp::now();
10124 for c in commands {
10125 state.jobs.push(JobRecord {
10126 node: node.to_owned(),
10127 round,
10128 seat: seat.to_owned(),
10129 id: c.id.clone(),
10130 description: c.description.clone(),
10131 checked_at,
10132 status: match c.exit_code {
10133 Some(0) => JobStatus::Completed,
10134 Some(_) => JobStatus::Failed,
10135 None => JobStatus::Unknown,
10136 },
10137 exit_code: c.exit_code,
10138 result_summary: c.result_summary.clone(),
10139 source: c.source.clone(),
10140 });
10141 }
10142}
10143
10144fn round_is_clean(
10165 blocking: usize,
10166 e2e_ok: bool,
10167 answered: usize,
10168 expected: usize,
10169 quota_missing: usize,
10170 policy: IncompleteReviewPolicy,
10171) -> bool {
10172 if blocking != 0 || !e2e_ok {
10173 return false;
10174 }
10175 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10176 return true;
10177 }
10178 answered > 0 && expected - answered <= quota_missing
10179}
10180
10181pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10205 if max_rounds == 0
10206 || reviews.last().is_some_and(|r| {
10207 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10208 })
10209 {
10210 return Some(RunStatus::Gating);
10211 }
10212 let last = reviews.last()?;
10213 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10214 if reviews.len() < max_rounds && !stagnant {
10215 return None;
10216 }
10217 if last.incomplete() && last.blocking == 0 {
10218 return Some(RunStatus::Blocked);
10219 }
10220 if last.e2e_status() == E2eStatus::ResourceBlocked
10221 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10222 {
10223 return None;
10224 }
10225 Some(if last.e2e.iter().all(CommandOutcome::ok) {
10226 RunStatus::Gating
10227 } else {
10228 RunStatus::Blocked
10229 })
10230}
10231
10232pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10247 if nudged {
10248 (full / 4).max(Duration::from_secs(120)).min(full)
10249 } else {
10250 full
10251 }
10252}
10253
10254#[allow(clippy::too_many_arguments)]
10283async fn ask_json_wave<T>(
10284 jobs: Vec<SeatJob>,
10285 sem: Arc<Semaphore>,
10286 retries: usize,
10287 roster: &[AgentSpec],
10288 ctx: &WaveCtx<'_>,
10289 losses: &mut Vec<QuotaLoss>,
10290 state: &mut RunState,
10291 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10292) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10293where
10294 T: serde::de::DeserializeOwned + Send + 'static,
10295{
10296 ask_wave_with(
10297 jobs,
10298 sem,
10299 retries,
10300 roster,
10301 ctx,
10302 losses,
10303 state,
10304 &|text: &str| {
10305 let v = verdict::extract_json::<T>(text)?;
10306 validate(&v)?;
10307 Ok(v)
10308 },
10309 )
10310 .await
10311}
10312
10313#[allow(clippy::too_many_arguments)]
10322async fn ask_wave_with<T>(
10323 jobs: Vec<SeatJob>,
10324 sem: Arc<Semaphore>,
10325 retries: usize,
10326 roster: &[AgentSpec],
10327 ctx: &WaveCtx<'_>,
10328 losses: &mut Vec<QuotaLoss>,
10329 state: &mut RunState,
10330 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10331) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10332where
10333 T: Send + 'static,
10334{
10335 let n = jobs.len();
10336 let originals: Vec<SeatJob> = jobs;
10337 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10338 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10339 let mut nudges: Vec<usize> = vec![0; n];
10346 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10350 let mut tried: Vec<BTreeSet<String>> = specs
10351 .iter()
10352 .map(|s| BTreeSet::from([s.id.clone()]))
10353 .collect();
10354 let starts: Vec<usize> = specs
10355 .iter()
10356 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10357 .collect();
10358 let carry = ctx.carry_seats && !roster.is_empty();
10359 let carried: Vec<BTreeSet<String>> = originals
10362 .iter()
10363 .map(|j| {
10364 state
10365 .seat_history
10366 .get(&j.seat.key)
10367 .filter(|_| carry)
10368 .map(|h| h.failed.clone())
10369 .unwrap_or_default()
10370 })
10371 .collect();
10372 let mut prev: Vec<Option<FailClass>> = originals
10373 .iter()
10374 .map(|j| {
10375 state
10376 .seat_history
10377 .get(&j.seat.key)
10378 .filter(|_| carry)
10379 .and_then(|h| h.last_fail.clone())
10380 })
10381 .collect();
10382 let next_agent =
10383 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10384 let others: BTreeSet<String> = specs
10385 .iter()
10386 .enumerate()
10387 .filter(|(j, _)| *j != i)
10388 .map(|(_, s)| s.id.clone())
10389 .collect();
10390 pick_successor(
10391 roster,
10392 starts[i],
10393 tried,
10394 carry.then(|| &carried[i]),
10395 &others,
10396 )
10397 .cloned()
10398 };
10399 let mut fresh: Vec<Option<String>> = vec![None; n];
10400 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10401 let mut pending: Vec<usize> = (0..n).collect();
10402
10403 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10408 for round in 0..max_rounds {
10409 if pending.is_empty() {
10410 break;
10411 }
10412 let mut batch = Vec::with_capacity(pending.len());
10413 let mut renudged: Vec<&str> = Vec::new();
10414 for &i in &pending {
10415 let src = &originals[i];
10416 let full: &str = match &src.handover {
10421 Some(h) if specs[i].id != src.spec.id => h,
10422 _ => &src.prompt,
10423 };
10424 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10427 (
10428 full.to_owned(),
10429 src.timeout,
10430 format!("{}-{word}-{}", src.stem, specs[i].id),
10431 )
10432 } else if nudges[i] == 0 {
10433 (full.to_owned(), src.timeout, src.stem.clone())
10434 } else {
10435 renudged.push(src.seat.key.as_str());
10436 let why = done[i]
10437 .as_ref()
10438 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10439 .unwrap_or_else(|| "no parsable answer".to_owned());
10440 let nudge = prompt::nudge(&why);
10441 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10442 let prompt = if nudged {
10443 nudge
10444 } else {
10445 format!("{full}\n\n---\n\n{nudge}")
10446 };
10447 (
10448 prompt,
10449 retry_budget(src.timeout, nudged),
10450 format!("{}-retry{}", src.stem, nudges[i]),
10451 )
10452 };
10453 batch.push(SeatJob {
10454 spec: specs[i].clone(),
10455 seat: seats[i].clone(),
10456 cwd: src.cwd.clone(),
10457 prompt,
10458 timeout,
10459 allow_write: src.allow_write,
10460 sessions: src.sessions,
10461 artifacts: src.artifacts.clone(),
10462 stem,
10463 handover: None,
10464 });
10465 }
10466
10467 if !renudged.is_empty() {
10468 state.event(
10469 ctx.node,
10470 format!("retry {round}: re-asking {}", renudged.join(", ")),
10471 );
10472 }
10473 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10474 let mut still = Vec::new();
10475 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10476 seats[i] = seat;
10477 if let AgentOutcome::ResourceBlocked(why) = &out {
10480 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10481 continue;
10482 }
10483 let class = FailClass::of(&out);
10484 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10487 && nudges[i] < retries
10488 && !roster.is_empty();
10489 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10490 let next = should_hand_over(prev[i].as_ref(), &cur)
10491 .then(|| next_agent(i, &tried[i], &specs))
10492 .flatten();
10493 if carry {
10494 let h = state
10495 .seat_history
10496 .entry(originals[i].seat.key.clone())
10497 .or_default();
10498 h.failed.insert(specs[i].id.clone());
10499 h.last_fail = Some(cur.clone());
10500 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10501 h.last_ok = None;
10502 }
10503 if let Err(e) = state.save() {
10506 tracing::warn!("could not persist a seat's failure history: {e:#}");
10507 }
10508 }
10509 if let Some(next) = next {
10510 record_handover(
10511 state,
10512 ctx.node,
10513 &originals[i].seat.key,
10514 &specs[i].id,
10515 &next.id,
10516 &cur,
10517 &fail_reason(&out),
10518 );
10519 tried[i].insert(next.id.clone());
10520 prev[i] = Some(cur.clone());
10521 seats[i] =
10522 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10523 specs[i] = next;
10524 fresh[i] = Some(cur.stem_word().to_owned());
10525 nudges[i] = 0;
10526 done[i] = Some(Err(anyhow::anyhow!(
10527 "handed over after: {}",
10528 fail_reason(&out)
10529 )));
10530 still.push(i);
10531 continue;
10532 }
10533 }
10534 if carry
10535 && !nudge_first
10536 && let Some(cur) = class.clone()
10537 {
10538 let h = state
10540 .seat_history
10541 .entry(originals[i].seat.key.clone())
10542 .or_default();
10543 h.failed.insert(specs[i].id.clone());
10544 h.last_fail = Some(cur);
10545 }
10546 let parsed = match out {
10547 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10548 AgentOutcome::Quota(o) => {
10549 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10550 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10551 }
10552 AgentOutcome::Dropped(o) => {
10556 let why = o
10557 .dropped
10558 .as_ref()
10559 .map(|d| d.why.as_str())
10560 .unwrap_or("the CLI ended the stream without delivering its answer");
10561 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10562 }
10563 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10564 Err(anyhow::anyhow!(e))
10565 }
10566 };
10567 let quota = class == Some(FailClass::Quota);
10568 let failed = parsed.is_err();
10569 done[i] = Some(parsed);
10570 if carry && !failed {
10571 let h = state
10572 .seat_history
10573 .entry(originals[i].seat.key.clone())
10574 .or_default();
10575 h.failed.remove(&specs[i].id);
10576 h.last_ok = Some(specs[i].id.clone());
10577 h.last_fail = None;
10578 }
10579 let agent_failure = class.is_some()
10587 && !nudge_first
10588 && !roster.is_empty()
10589 && next_agent(i, &tried[i], &specs).is_some();
10590 if failed && !quota && !agent_failure && nudges[i] < retries {
10591 nudges[i] += 1;
10592 still.push(i);
10593 }
10594 }
10595 pending = still;
10596 }
10597
10598 for (i, q) in last_quota.into_iter().enumerate() {
10601 if let Some(reset) = q {
10602 losses.push(QuotaLoss {
10603 seat: originals[i].seat.key.clone(),
10604 node: ctx.node.to_owned(),
10605 at: Timestamp::now(),
10606 reset,
10607 });
10608 }
10609 }
10610
10611 seats
10612 .into_iter()
10613 .zip(done)
10614 .zip(nudges)
10615 .map(|((seat, res), attempts)| {
10616 (
10617 seat,
10618 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10619 attempts,
10620 )
10621 })
10622 .collect()
10623}
10624
10625fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10627 results
10628 .iter()
10629 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10630}
10631
10632fn block_wave(
10636 jobs: Vec<SeatJob>,
10637 state: &mut RunState,
10638 node: &str,
10639 attempt: usize,
10640 why: &str,
10641) -> Vec<(usize, SeatState, AgentOutcome)> {
10642 state.resource_wait = None;
10643 state
10644 .active
10645 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10646 if let Err(e) = state.save() {
10647 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10648 }
10649 jobs.into_iter()
10650 .enumerate()
10651 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10652 .collect()
10653}
10654
10655async fn acquire_cache_lease(
10668 state: &mut RunState,
10669 cache_dir: &Path,
10670 owner: &crate::cache::Owner,
10671 budget: Duration,
10672 node: &str,
10673 context: &str,
10674 cancel: &(dyn Fn() -> bool + Sync),
10675) -> Result<Option<crate::cache::Guard>> {
10676 let home = crate::run::home();
10677 let started = Instant::now();
10678 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10679 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10680 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10681 Err(e) => {
10682 state.event(
10683 "verify",
10684 format!("{context}: could not check the shared build cache: {e:#}"),
10685 );
10686 if let Err(e2) = state.save() {
10687 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10688 }
10689 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10690 }
10691 };
10692 state.event(
10693 "verify",
10694 format!(
10695 "{context}: waiting for the shared build cache at {} ({})",
10696 cache_dir.display(),
10697 busy.describe()
10698 ),
10699 );
10700 state.resource_wait = Some(crate::run::ResourceWait {
10703 node: context.to_owned(),
10704 kind: "shared build cache".to_owned(),
10705 holder: busy.describe(),
10706 since: Timestamp::now(),
10707 limit_secs: budget.as_secs(),
10708 });
10709 if let Err(e) = state.save() {
10710 tracing::warn!("could not persist a cache wait: {e:#}");
10711 }
10712 let remaining = budget.saturating_sub(started.elapsed());
10713 let waited = crate::cache::wait_for_unless(
10714 &home,
10715 cache_dir,
10716 owner,
10717 remaining,
10718 Duration::from_secs(5),
10719 cancel,
10720 )
10721 .await;
10722 state.resource_wait = None;
10723 let cause = match &waited {
10724 Ok(Some(_)) => None,
10725 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10726 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10727 Some(crate::run::ResourceBlockCause::Timeout {
10728 waited_secs: started.elapsed().as_secs(),
10729 limit_secs: budget.as_secs(),
10730 })
10731 }
10732 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10733 };
10734 if let Some(cause) = cause {
10735 let detail = match &waited {
10736 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10737 Err(error) => format!("{error:#}"),
10738 };
10739 state.resource_block = Some(crate::run::ResourceBlock {
10740 cause,
10741 node: node.to_owned(),
10742 kind: "shared build cache".into(),
10743 detail,
10744 at: Timestamp::now(),
10745 });
10746 state.event(
10747 "resource",
10748 state.resource_block.as_ref().unwrap().description(),
10749 );
10750 }
10751 if let Err(error) = state.save() {
10752 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10753 }
10754 waited
10755}
10756
10757#[derive(Debug)]
10762struct CacheCheckError(String);
10763
10764impl std::fmt::Display for CacheCheckError {
10765 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10766 f.write_str(&self.0)
10767 }
10768}
10769
10770impl std::error::Error for CacheCheckError {}
10771
10772fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10774 state
10775 .resource_block
10776 .get_or_insert_with(|| crate::run::ResourceBlock {
10777 cause: crate::run::ResourceBlockCause::Other,
10778 node: node.to_owned(),
10779 kind: "shared build cache".into(),
10780 detail: format!("{error:#}"),
10781 at: Timestamp::now(),
10782 });
10783 if let Err(error) = state.save() {
10784 tracing::warn!("could not persist cache resource block: {error:#}");
10785 }
10786}
10787
10788#[allow(clippy::too_many_arguments)]
10809async fn with_cache_lease<'s, F, Fut>(
10810 state: &'s mut RunState,
10811 cache_dir: Option<&Path>,
10812 node: &str,
10813 seat: &str,
10814 worktree: &Path,
10815 head: &str,
10816 budget: Duration,
10817 context: &str,
10818 wait: &crate::repo_lock::WaitPolicy,
10819 body: F,
10820) -> Result<(Vec<CommandOutcome>, bool)>
10821where
10822 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10823 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10824{
10825 let started = Instant::now();
10828 let home = crate::run::home();
10829 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10830 let guard = if let Some(cache_dir) = cache_dir {
10831 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10832 match acquire_cache_lease(
10833 state,
10834 cache_dir,
10835 &owner,
10836 resource_limit,
10837 node,
10838 context,
10839 &|| wait.cancelled(),
10840 )
10841 .await
10842 .and_then(|g| {
10843 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10844 }) {
10845 Ok(g) => Some(g),
10846 Err(e) => {
10847 record_cache_error(state, node, &e);
10848 return Ok((
10849 vec![CommandOutcome {
10850 command: "(waiting for the shared build cache)".into(),
10851 code: None,
10852 output_tail: e.to_string(),
10853 duration_ms: started.elapsed().as_millis() as u64,
10854 resource_blocked: true,
10855 }],
10856 false,
10857 ));
10858 }
10859 }
10860 } else {
10861 None
10862 };
10863 let _coordination = coordinate_operation(state, worktree, wait).await?;
10864 if let Some(cache_dir) = cache_dir {
10865 let identity = crate::cache::Identity::new(worktree, head);
10866 if let Err(e) = crate::cache::ensure_fresh(
10867 &home,
10868 cache_dir,
10869 &identity,
10870 &state.config.verify.cargo_manifests(),
10871 ) {
10872 record_cache_error(state, node, &e);
10873 state.event(
10874 "verify",
10875 format!(
10876 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10877 worktree.display(),
10878 short(head)
10879 ),
10880 );
10881 if let Some(guard) = guard {
10882 guard.release();
10883 }
10884 return Ok((
10885 vec![CommandOutcome {
10886 command: "(confirming the shared build cache is fresh)".into(),
10887 code: None,
10888 output_tail: e.to_string(),
10889 duration_ms: started.elapsed().as_millis() as u64,
10890 resource_blocked: true,
10891 }],
10892 false,
10893 ));
10894 }
10895 }
10896 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10897 if !timed_out_pids.is_empty() {
10898 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10899 }
10900 drop(_coordination);
10903 if let Some(guard) = guard {
10904 guard.release();
10905 }
10906 Ok((outcomes, retried))
10907}
10908
10909async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10921 wait_for_pids_with(
10922 pids,
10923 crate::proc::pid_alive,
10924 LEASE_RELEASE_POLL,
10925 LEASE_RELEASE_MAX_WAIT,
10926 )
10927 .await;
10928}
10929
10930async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10936 pids: &[u32],
10937 alive: F,
10938 poll: Duration,
10939 max_wait: Duration,
10940) {
10941 let deadline = Instant::now() + max_wait;
10942 loop {
10943 if pids.iter().all(|&pid| !alive(pid)) {
10944 return;
10945 }
10946 if Instant::now() >= deadline {
10947 return;
10948 }
10949 tokio::time::sleep(poll).await;
10950 }
10951}
10952
10953fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10960 outcomes.iter().any(|o| o.resource_blocked)
10961}
10962
10963pub(crate) enum GateFix {
10965 Retry,
10967 Stop,
10970 Defer,
10973}
10974
10975fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10983 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10984 red.peek().is_some()
10985 && red.all(|o| {
10986 !o.resource_blocked
10987 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10988 && !o.output_tail.trim().is_empty()
10989 })
10990}
10991
10992fn e2e_outcome_label(o: &CommandOutcome) -> String {
10996 if o.ok() {
10997 return "pass".to_owned();
10998 }
10999 let reason = if o.build_failed() {
11000 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11001 } else {
11002 format!("FAIL ({:?})", o.code)
11003 };
11004 format!(
11005 "{reason}\n{}",
11006 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11007 )
11008}
11009
11010async fn run_e2e_with_retry(
11018 state: &mut RunState,
11019 shell: &[String],
11020 commands: &[String],
11021 worktree: &Path,
11022 timeout: Duration,
11023 context: &str,
11024) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11025 let (mut e2e, mut timed_out_pids) = run_commands(
11026 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11027 )
11028 .await;
11029 for o in &e2e {
11030 state.event(
11031 "verify",
11032 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11033 );
11034 }
11035 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11039 if verify_retried {
11040 state.event(
11041 "verify",
11042 format!(
11043 "{context}: verify could not build/link, not a test result — retrying once \
11044 before concluding"
11045 ),
11046 );
11047 let retried = run_commands(
11048 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11049 )
11050 .await;
11051 e2e = retried.0;
11052 timed_out_pids.extend(retried.1);
11055 for o in &e2e {
11056 state.event(
11057 "verify",
11058 format!(
11059 "{context}: retry `{}` -> {}",
11060 o.command,
11061 e2e_outcome_label(o)
11062 ),
11063 );
11064 }
11065 }
11066 (e2e, verify_retried, timed_out_pids)
11067}
11068
11069#[allow(clippy::too_many_arguments)]
11085async fn run_commands(
11086 state: &mut RunState,
11087 node: &str,
11088 task: &str,
11089 attempt: usize,
11090 shell: &[String],
11091 commands: &[String],
11092 cwd: &Path,
11093 timeout: Duration,
11094) -> (Vec<CommandOutcome>, Vec<u32>) {
11095 if commands.is_empty() {
11096 return (Vec::new(), Vec::new());
11101 }
11102 let mut out = Vec::new();
11103 let mut timed_out_pids = Vec::new();
11104 let total = commands.len();
11105 for (idx, command) in commands.iter().enumerate() {
11106 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11107 if let Err(e) = state.save() {
11108 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11109 }
11110 let started = Instant::now();
11111 let mut cmd = tokio::process::Command::new(&shell[0]);
11112 cmd.quiet();
11113 for key in crate::worker::INVOCATION_ENV {
11117 cmd.env_remove(key);
11118 }
11119 cmd.args(&shell[1..])
11120 .arg(command)
11121 .current_dir(cwd)
11122 .stdin(std::process::Stdio::null())
11123 .stdout(std::process::Stdio::piped())
11124 .stderr(std::process::Stdio::piped())
11125 .kill_on_drop(true);
11126 let mut outputs = Vec::new();
11127 let protected = (|| -> Result<()> {
11128 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11129 outputs.push(guard);
11130 }
11131 if let Some(cache) = state.config.cache_dir() {
11132 outputs.push(crate::retention::OutputLease::cache(
11133 &crate::run::home(),
11134 &cache,
11135 false,
11136 )?);
11137 }
11138 Ok(())
11139 })();
11140 if let Err(e) = protected {
11141 out.push(CommandOutcome {
11142 command: command.clone(),
11143 code: None,
11144 output_tail: format!("output ownership unavailable: {e:#}"),
11145 duration_ms: 0,
11146 resource_blocked: true,
11147 });
11148 continue;
11149 }
11150 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11151 let (code, body) = match spawned {
11152 Ok(mut running) => {
11153 let pid = running.child().id();
11154 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11155 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11156 let mut paths = vec![cwd.to_owned()];
11157 if let Some(cache) = state.config.cache_dir() {
11158 paths.push(cache);
11159 }
11160 let mut pressure = None;
11161 let status = tokio::select! {
11162 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11163 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11164 pressure = Some(reason); None
11165 }
11166 };
11167 let (code, failure) = match status {
11168 Some(Ok(Ok(status))) => (
11169 status.code(),
11170 status
11171 .code()
11172 .is_none()
11173 .then(|| format!("command terminated: {status}")),
11174 ),
11175 Some(Ok(Err(error))) => {
11176 (None, Some(format!("failed waiting for command: {error}")))
11177 }
11178 Some(Err(_)) => {
11179 if !running.coordinated()
11183 && let Some(pid) = pid
11184 {
11185 timed_out_pids.push(pid);
11186 }
11187 (
11188 None,
11189 Some(format!("timed out after {}s", timeout.as_secs())),
11190 )
11191 }
11192 None => (None, None),
11193 };
11194 running.finish().await;
11195 if let Some(reason) = pressure {
11196 state.resource_block = Some(crate::run::ResourceBlock {
11197 cause: crate::run::ResourceBlockCause::Other,
11198 node: node.to_owned(),
11199 kind: "disk pressure".to_owned(),
11200 detail: reason.clone(),
11201 at: Timestamp::now(),
11202 });
11203 out.push(CommandOutcome {
11204 command: command.clone(),
11205 code: None,
11206 output_tail: reason,
11207 duration_ms: started.elapsed().as_millis() as u64,
11208 resource_blocked: true,
11209 });
11210 break;
11211 }
11212 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11213 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11214 if let Some(failure) = failure {
11215 body.push('\n');
11216 body.push_str(&failure);
11217 }
11218 (code, body)
11219 }
11220 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11221 };
11222 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11223 if !body.is_empty() {
11224 match crate::run::write_verification_artifact(state, task, &body) {
11228 Ok(name) if summary != body => {
11229 summary = format!("[full output: artifacts/{name}]\n{summary}");
11230 }
11231 Ok(_) => {}
11232 Err(e) => {
11233 let warning = format!("could not save the full {task} output: {e:#}");
11234 tracing::warn!("{warning}");
11235 state.event(node, warning);
11236 if summary != body {
11237 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11238 }
11239 }
11240 }
11241 }
11242 out.push(CommandOutcome {
11243 command: command.clone(),
11244 code,
11245 output_tail: summary,
11246 duration_ms: started.elapsed().as_millis() as u64,
11247 resource_blocked: false,
11248 });
11249 }
11250 state.task_finished(task);
11251 if let Err(e) = state.save() {
11252 tracing::warn!("could not persist the end of {task}: {e:#}");
11253 }
11254 (out, timed_out_pids)
11255}
11256
11257fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11269 let repo = repo.display();
11270 match style {
11271 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11272 MergeStyle::Squash => {
11273 let subject = message
11276 .lines()
11277 .next()
11278 .unwrap_or(branch)
11279 .replace(['\\', '"', '$', '`'], "");
11280 format!(
11281 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11282 )
11283 }
11284 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11285 }
11286}
11287
11288const PR_TITLE_MAX: usize = 240;
11299
11300fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11311 let line = queue::first_line(text)?;
11312 let chars: Vec<char> = line.chars().collect();
11313 let (kept, mark) = if chars.len() <= max {
11314 (line.to_owned(), "")
11315 } else if let Some(end) = sentence_end(&chars, max) {
11316 (chars[..end].iter().collect::<String>(), "")
11317 } else {
11318 let room = max.saturating_sub(3);
11319 let cut = if chars[room].is_whitespace() {
11322 room
11323 } else {
11324 chars[..room]
11325 .iter()
11326 .rposition(|c| c.is_whitespace())
11327 .unwrap_or(room)
11328 };
11329 let head: String = chars[..cut].iter().collect();
11330 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11331 (head.to_owned(), "...")
11332 };
11333 if kept.is_empty() {
11334 return None;
11335 }
11336 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11337 return None;
11338 }
11339 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11340 return None;
11341 }
11342 Some(format!("{kept}{mark}"))
11343}
11344
11345fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11349 const MIN: usize = 20;
11350 for i in MIN..max.min(chars.len()) {
11351 if !matches!(chars[i], '.' | '!' | '?') {
11352 continue;
11353 }
11354 let Some(&next) = chars.get(i + 1) else {
11355 continue;
11356 };
11357 if !next.is_whitespace() {
11358 continue;
11359 }
11360 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11361 if after.is_some_and(|c| c.is_ascii_lowercase()) {
11362 continue;
11363 }
11364 let word: String = chars[..i]
11365 .iter()
11366 .rev()
11367 .take_while(|c| !c.is_whitespace())
11368 .collect::<Vec<_>>()
11369 .into_iter()
11370 .rev()
11371 .collect();
11372 let word = word.to_ascii_lowercase();
11373 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11374 continue;
11375 }
11376 let end = chars[..i]
11377 .iter()
11378 .rposition(|c| !c.is_whitespace())
11379 .map_or(i, |p| p + 1);
11380 return Some(end);
11381 }
11382 None
11383}
11384
11385struct PrMessage {
11390 title: String,
11391 body: String,
11392}
11393
11394impl PrMessage {
11395 fn commit_message(&self) -> String {
11399 format!("{}\n\n{}", self.title, self.body)
11400 }
11401}
11402
11403fn title_marker(line: &str) -> Option<&str> {
11405 let line = line.trim();
11406 let head = line.get(..6)?;
11407 head.eq_ignore_ascii_case("title:")
11408 .then(|| line[6..].trim())
11409}
11410
11411fn summary_title(summary: &str) -> Option<String> {
11416 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11417 let raw = title_marker(first)?;
11418 if raw.is_empty() {
11419 return None;
11420 }
11421 let title = queue::title_from(raw, PR_TITLE_MAX);
11422 let lower = title.to_ascii_lowercase();
11423 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11424 return None;
11425 }
11426 Some(title)
11427}
11428
11429const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11431
11432pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11434
11435pub(crate) fn is_review_run(state: &RunState) -> bool {
11439 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11440}
11441
11442fn review_title(state: &RunState) -> Option<String> {
11447 english_subject(
11448 state.reviewed_commits.as_ref()?.first()?,
11449 state.config.graph.github_english(),
11450 )
11451}
11452
11453fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11456 let raw = raw.trim();
11457 if raw.is_empty() {
11458 return None;
11459 }
11460 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11461 return None;
11462 }
11463 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11464 return None;
11465 }
11466 let title = queue::title_from(raw, PR_TITLE_MAX);
11467 let lower = title.to_ascii_lowercase();
11468 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11469 return None;
11470 }
11471 Some(title)
11472}
11473
11474#[derive(Debug, Clone, PartialEq, Eq)]
11477struct BranchFacts {
11478 commits: Vec<(String, String)>,
11480 stat: String,
11482}
11483
11484const STAT_FILE_LINES: usize = 25;
11486const COMMITS_MAX_CHARS: usize = 20_000;
11488
11489async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11492 let commits = git::commit_log(repo, base, branch).await.ok()?;
11493 if commits.is_empty() {
11494 return None;
11495 }
11496 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11497 let lines: Vec<&str> = stat.lines().collect();
11498 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11499 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11500 let more = format!(" ... {omitted} more file(s)");
11501 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11502 kept.push(&more);
11503 kept.push(lines[lines.len() - 1]);
11504 kept.join("\n")
11505 } else {
11506 lines.join("\n")
11507 };
11508 Some(BranchFacts { commits, stat })
11509}
11510
11511fn markdown_safe(text: &str) -> String {
11514 text.replace("</details>", "</details>")
11515 .replace("\x60\x60\x60", "~~~")
11516}
11517
11518fn neutral_title(state: &RunState, winner: char) -> String {
11519 format!(
11520 "chore: land candidate {} of run {}",
11521 winner.to_ascii_uppercase(),
11522 state.id
11523 )
11524}
11525
11526pub(crate) async fn repair_landing_gate(
11529 state: &mut RunState,
11530 winner: &Candidate,
11531 outcomes: &[CommandOutcome],
11532) -> Result<GateFix> {
11533 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11534 return Ok(GateFix::Stop);
11535 }
11536 let mut runner = Runner {
11537 roles: state.config.resolve_roles()?,
11538 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11539 state: state.clone(),
11540 pause: Pause::new(),
11541 interrupt: Pause::new(),
11542 };
11543 let result = runner.gate_fix_round(winner, outcomes).await;
11544 *state = runner.state;
11545 result
11546}
11547
11548pub(crate) async fn verify_current_head(
11551 state: &mut RunState,
11552 winner: &Candidate,
11553 wait: &crate::repo_lock::WaitPolicy,
11554) -> Result<bool> {
11555 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11556 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11558 state
11559 .verification_history
11560 .push(crate::run::VerificationEvidence {
11561 superseded_by: Some(head.clone()),
11562 archived_at: Some(Timestamp::now()),
11563 fixes_spent: Some(state.gate_fixes.len()),
11564 gate_head: None,
11565 gate_ran: false,
11566 gate: Vec::new(),
11567 e2e: state.head_e2e.take(),
11568 });
11569 }
11570 if let Some(e) = &state.head_e2e
11571 && e.head == head
11572 {
11573 if e.passed {
11574 return Ok(true);
11575 }
11576 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11577 state.status = RunStatus::Blocked;
11578 state.save()?;
11579 return Ok(false);
11580 }
11581 } else if state.reviews.last().is_some_and(|r| {
11582 r.verified_head.as_deref() == Some(head.as_str())
11583 && (r.e2e_status() == E2eStatus::Passed
11584 || (r.e2e_status() == E2eStatus::NotConfigured
11585 && state.config.verify.e2e.is_empty()))
11586 }) || state
11587 .gate_repair_e2e
11588 .as_ref()
11589 .is_some_and(|e| e.head == head && e.passed)
11590 {
11591 return Ok(true);
11592 }
11593 if let Some(e) = &state.gate_repair_e2e
11594 && e.head == head
11595 && !e.passed
11596 && !e.outcomes.is_empty()
11597 && !verify_inconclusive(&e.outcomes)
11598 {
11599 state.status = RunStatus::Blocked;
11600 state.save()?;
11601 return Ok(false);
11602 }
11603 state.head_e2e = Some(crate::run::GateRepairE2e {
11604 head: head.clone(),
11605 passed: false,
11606 outcomes: Vec::new(),
11607 });
11608 state.save()?;
11609 let commands = state.config.verify.e2e.clone();
11610 let shell = state.config.shell();
11611 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11612 let cache_dir = state.config.cache_dir();
11613 let outcomes = if commands.is_empty() {
11614 Vec::new()
11615 } else {
11616 let (outcomes, _) = with_cache_lease(
11617 state,
11618 cache_dir.as_deref(),
11619 "e2e",
11620 "e2e",
11621 &winner.worktree,
11622 &head,
11623 timeout,
11624 "current head e2e",
11625 wait,
11626 |state, budget| async move {
11627 run_e2e_with_retry(
11628 state,
11629 &shell,
11630 &commands,
11631 &winner.worktree,
11632 budget,
11633 "current head e2e",
11634 )
11635 .await
11636 },
11637 )
11638 .await?;
11639 outcomes
11640 };
11641 let unresolved = verify_inconclusive(&outcomes);
11642 let passed = !unresolved
11643 && outcomes.iter().all(CommandOutcome::ok)
11644 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11645 state.head_e2e = Some(crate::run::GateRepairE2e {
11646 head: head.clone(),
11647 passed,
11648 outcomes,
11649 });
11650 if !passed && !unresolved {
11651 state.status = RunStatus::Blocked;
11652 state.event(
11653 "e2e",
11654 format!("verify.e2e failed on head {head}; nothing published"),
11655 );
11656 }
11657 state.save()?;
11658 Ok(passed)
11659}
11660
11661pub(crate) async fn check_gate_repair_e2e(
11664 state: &mut RunState,
11665 winner: &Candidate,
11666 wait: &crate::repo_lock::WaitPolicy,
11667) -> Result<GateFix> {
11668 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11669 return Ok(GateFix::Retry);
11670 }
11671 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11672 if state
11673 .gate_repair_e2e
11674 .as_ref()
11675 .is_some_and(|r| r.head == head && r.passed)
11676 {
11677 return Ok(GateFix::Retry);
11678 }
11679 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11680 head: head.clone(),
11681 passed: false,
11682 outcomes: Vec::new(),
11683 });
11684 state.save()?;
11685 let commands = state.config.verify.e2e.clone();
11686 let outcomes = if commands.is_empty() {
11687 Vec::new()
11688 } else {
11689 let shell = state.config.shell();
11690 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11691 let cache_dir = state.config.cache_dir();
11692 let (outcomes, _) = with_cache_lease(
11693 state,
11694 cache_dir.as_deref(),
11695 "e2e",
11696 "e2e",
11697 &winner.worktree,
11698 &head,
11699 timeout,
11700 "gate repair e2e",
11701 wait,
11702 |state, budget| async move {
11703 run_e2e_with_retry(
11704 state,
11705 &shell,
11706 &commands,
11707 &winner.worktree,
11708 budget,
11709 "gate repair e2e",
11710 )
11711 .await
11712 },
11713 )
11714 .await?;
11715 outcomes
11716 };
11717 let deferred = verify_inconclusive(&outcomes);
11718 let passed = outcomes.iter().all(CommandOutcome::ok)
11719 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11720 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11721 head,
11722 passed,
11723 outcomes,
11724 });
11725 state.save()?;
11726 if deferred {
11727 Ok(GateFix::Defer)
11728 } else if !passed {
11729 state.event(
11730 "gate",
11731 "verify.e2e failed after gate repair; nothing published",
11732 );
11733 Ok(GateFix::Stop)
11734 } else {
11735 Ok(GateFix::Retry)
11736 }
11737}
11738
11739pub(crate) async fn run_landing_gate(
11741 state: &mut RunState,
11742 winner: &Candidate,
11743 wait: &crate::repo_lock::WaitPolicy,
11744) -> Result<Vec<CommandOutcome>> {
11745 let shell = state.config.shell();
11746 let gate_commands = state.config.verify.gate.clone();
11747 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11748 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11749 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
11750 Vec::new()
11751 } else {
11752 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11753 let cache_dir = state.config.cache_dir();
11754 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11755 let (outcomes, _) = with_cache_lease(
11756 state,
11757 cache_dir.as_deref(),
11758 "gate",
11759 "gate",
11760 &winner.worktree,
11761 &head,
11762 timeout,
11763 "final gate",
11764 wait,
11765 |state, budget| {
11766 let shell = shell.clone();
11767 let gate_commands = gate_commands.clone();
11768 let winner = winner.clone();
11769 async move {
11770 Runner::run_pre_gate(state, &winner).await;
11771 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
11773 let (outcomes, timed_out_pids) = run_commands(
11774 state,
11775 "gate",
11776 "gate",
11777 0,
11778 &shell,
11779 &gate_commands,
11780 &winner.worktree,
11781 budget,
11782 )
11783 .await;
11784 (outcomes, false, timed_out_pids)
11785 }
11786 },
11787 )
11788 .await?;
11789 outcomes
11790 };
11791 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
11792 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
11793 let mut outcomes = outcomes;
11794 outcomes.push(CommandOutcome {
11795 command: "gate head stability".into(), code: None,
11796 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
11797 duration_ms: 0, resource_blocked: false,
11798 });
11799 return Ok(outcomes);
11800 }
11801 if outcomes.is_empty() {
11802 state.event("gate", "no gate commands configured; not tested");
11807 }
11808 for o in &outcomes {
11809 state.event(
11810 "gate",
11811 format!(
11812 "`{}` -> {}",
11813 o.command,
11814 if o.ok() {
11815 "pass".to_owned()
11816 } else {
11817 format!(
11818 "FAIL ({:?})\n{}",
11819 o.code,
11820 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11821 )
11822 }
11823 ),
11824 );
11825 }
11826 Ok(outcomes)
11827}
11828
11829pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11835 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11836 ""
11837 } else {
11838 pr_title
11839 }
11840}
11841
11842pub fn landing_subject_source(state: &RunState) -> String {
11846 if is_review_run(state) {
11847 let winner = state.candidates.first().map_or('A', |c| c.label);
11848 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11849 }
11850 state.instruction.clone()
11851}
11852
11853fn summary_without_title(summary: &str) -> String {
11856 let mut lines = summary.trim().lines().peekable();
11857 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11858 lines.next();
11859 }
11860 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11861}
11862
11863#[cfg(test)]
11875fn pr_message(state: &RunState, winner: char) -> PrMessage {
11876 pr_message_with(state, winner, None)
11877}
11878
11879#[cfg(test)]
11882fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11883 let raw = pr_message_raw(
11884 state,
11885 winner,
11886 facts,
11887 crate::pr_outcome::FinalHead::Unreported,
11888 ChangeNote::NotAsked,
11889 );
11890 let id = crate::scrub::Identity::current();
11891 PrMessage {
11892 title: crate::scrub::scrub(&raw.title, &id),
11893 body: crate::scrub::scrub(&raw.body, &id),
11894 }
11895}
11896
11897#[derive(Debug, Clone, Copy)]
11899enum ChangeNote<'a> {
11900 NotAsked,
11902 Summary(&'a crate::run::ChangeSummary),
11904 Unavailable(&'a str),
11906}
11907
11908fn pr_message_raw(
11909 state: &RunState,
11910 winner: char,
11911 facts: Option<&BranchFacts>,
11912 head: crate::pr_outcome::FinalHead<'_>,
11913 change: ChangeNote<'_>,
11914) -> PrMessage {
11915 if !matches!(change, ChangeNote::NotAsked) {
11916 let (title, mut body) = match change {
11917 ChangeNote::Summary(c) => (
11918 c.title.clone(),
11919 format!("## Summary\n\n{}\n\n", c.body.trim()),
11920 ),
11921 ChangeNote::Unavailable(why) => (
11922 neutral_title(state, winner),
11923 format!(
11924 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11925 ),
11926 ),
11927 ChangeNote::NotAsked => unreachable!(),
11928 };
11929 body.push_str(&public_pr_evidence(state, head));
11930 body.push_str(&format!(
11931 "\n---\nmagi:run/{} magi:candidate-{}\n",
11932 state.id,
11933 winner.to_ascii_lowercase()
11934 ));
11935 return PrMessage { title, body };
11936 }
11937 pr_preview(state, winner, facts, head)
11938}
11939
11940fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11943 let mut body = crate::pr_outcome::review_rounds(state);
11944 body.push_str(&crate::pr_outcome::final_verification(state, head));
11945 let open = state.last_round_findings();
11946 if !open.is_empty() {
11947 body.push_str("## Open review findings\n\n");
11948 for f in open {
11949 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11950 }
11951 body.push('\n');
11952 }
11953 if let Some(c) = state.change_summary.as_ref()
11954 && head == crate::pr_outcome::FinalHead::At(&c.head)
11955 {
11956 body.push_str(&format!(
11957 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11958 c.base, c.head
11959 ));
11960 }
11961 body
11962}
11963
11964fn pr_preview(
11966 state: &RunState,
11967 winner: char,
11968 facts: Option<&BranchFacts>,
11969 head: crate::pr_outcome::FinalHead<'_>,
11970) -> PrMessage {
11971 let summary = state
11972 .candidates
11973 .iter()
11974 .find(|c| c.label == winner)
11975 .map(|c| c.summary.as_str())
11976 .unwrap_or_default();
11977 let review = is_review_run(state);
11980 let title = if review {
11981 facts
11982 .and_then(|f| {
11983 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11984 })
11985 .or_else(|| review_title(state))
11986 .or_else(|| {
11987 state
11988 .candidates
11989 .iter()
11990 .find(|c| c.label == winner)
11991 .filter(|c| !c.branch.starts_with("magi/"))
11992 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11993 })
11994 .unwrap_or_else(|| neutral_title(state, winner))
11995 } else {
11996 summary_title(summary).unwrap_or_else(|| {
11997 english_title(
11998 &state.instruction,
11999 PR_TITLE_MAX,
12000 state.config.graph.github_english(),
12001 )
12002 .unwrap_or_else(|| neutral_title(state, winner))
12003 })
12004 };
12005
12006 let mut body = String::new();
12007 let what = summary_without_title(summary);
12008 if !what.is_empty() {
12009 body.push_str("## Summary\n\n");
12010 body.push_str(&what);
12011 body.push_str("\n\n");
12012 }
12013
12014 body.push_str(&crate::pr_outcome::review_rounds(state));
12016 body.push_str(&crate::pr_outcome::final_verification(state, head));
12017 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12018
12019 let open = state.open_findings();
12020 if !open.is_empty() {
12021 body.push_str("## Open review findings\n\n");
12022 for f in &open {
12023 body.push_str(&format!(
12024 "- `{}` [{:?}] {}\n",
12025 f.id, f.severity, "Context retained in local review records."
12026 ));
12027 }
12028 body.push('\n');
12029 }
12030
12031 if let Some(fix) = fix
12032 && !fix.rejected.is_empty()
12033 {
12034 body.push_str("## Declined by the fixer\n\n");
12035 for r in &fix.rejected {
12036 body.push_str(&format!(
12037 "- `{}`: {}\n",
12038 r.id, "Reason retained in local review records."
12039 ));
12040 }
12041 body.push('\n');
12042 }
12043
12044 if review {
12045 body.push_str("## Commits under review\n\n");
12047 if let Some(facts) = facts {
12048 let mut left = COMMITS_MAX_CHARS;
12049 for (i, (subject, text)) in facts.commits.iter().enumerate() {
12050 let mut entry = format!("- {}\n", markdown_safe(subject));
12051 for l in markdown_safe(text).lines() {
12052 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
12053 }
12054 if left == 0 {
12055 body.push_str(&format!(
12056 "- ... {} more commit(s)\n",
12057 facts.commits.len() - i
12058 ));
12059 break;
12060 }
12061 if entry.len() > left {
12062 let mut end = left;
12065 while !entry.is_char_boundary(end) {
12066 end -= 1;
12067 }
12068 entry.truncate(end);
12069 entry.push_str("\n ... (truncated)\n");
12070 left = 0;
12071 } else {
12072 left -= entry.len();
12073 }
12074 body.push_str(&entry);
12075 }
12076 if !facts.stat.trim().is_empty() {
12077 body.push_str(&format!(
12078 "\n## Diff stat\n\n```\n{}\n```\n",
12079 markdown_safe(facts.stat.trim())
12080 ));
12081 }
12082 } else {
12083 match &state.reviewed_commits {
12084 Some(subjects) => {
12085 for s in subjects {
12086 body.push_str(&format!("- {}\n", s.trim()));
12087 }
12088 }
12089 None => {
12090 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12093 body.push_str(rest.trim());
12094 body.push('\n');
12095 }
12096 }
12097 }
12098 } else {
12099 let task = state.instruction.trim();
12100 let task = if task.is_empty() {
12101 "(empty task)"
12102 } else {
12103 task
12104 };
12105 body.push_str(&format!(
12106 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12107 task.replace("</details>", "</details>")
12108 ));
12109 }
12110
12111 body.push_str(&format!(
12112 "\n---\nmagi:run/{} magi:candidate-{}\n",
12113 state.id,
12114 winner.to_ascii_lowercase()
12115 ));
12116
12117 PrMessage { title, body }
12118}
12119
12120fn seeded_instruction(state: &RunState) -> String {
12123 let mut inputs = Vec::new();
12124 if let Some(start) = &state.sources.start_from {
12125 inputs.push(format!(
12126 "Start from `{}` at `{}` (including ancestry).",
12127 start.token, start.sha
12128 ));
12129 }
12130 for pick in &state.sources.pick_commits {
12131 inputs.push(format!(
12132 "Pick `{}` at `{}` (single commit).",
12133 pick.token, pick.sha
12134 ));
12135 }
12136 if inputs.is_empty() {
12137 state.instruction.clone()
12138 } else {
12139 format!(
12140 "{}\n\n# Explicit source inputs already prepared\n\n{}",
12141 state.instruction,
12142 inputs.join("\n")
12143 )
12144 }
12145}
12146
12147async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12151 let base = &state.base_branch;
12152 let mut against = base.clone();
12153 if mode == MergeMode::Pr {
12154 let remote = &state.config.merge.remote;
12158 let tracking = format!("{remote}/{base}");
12159 let fetched = git::fetch(repo, remote, base).await;
12160 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12161 against = tracking;
12162 } else {
12163 return false;
12164 }
12165 }
12166 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12167}
12168
12169fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12172 let mut detail = format!(
12173 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12174 nothing to open a pull request for"
12175 );
12176 detail.push_str(
12177 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12178 );
12179 detail
12180}
12181
12182#[derive(Debug, PartialEq, Eq)]
12185enum PrPlan {
12186 Create,
12187 Adopt { url: String, title: String },
12188 Stop(String),
12189}
12190
12191fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12194 match found {
12195 Ok(land::OpenPr::None) => PrPlan::Create,
12196 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12197 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12198 "several open pull requests exist for this branch, not picking one: {}",
12199 urls.join(" ")
12200 )),
12201 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12202 }
12203}
12204
12205pub(crate) async fn gh_pr_create(
12207 cwd: &Path,
12208 base: &str,
12209 head: &str,
12210 title: &str,
12211 body: &str,
12212) -> Result<String> {
12213 let out = tokio::process::Command::new("gh")
12214 .args([
12215 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12216 ])
12217 .current_dir(cwd)
12218 .quiet()
12219 .stdin(std::process::Stdio::null())
12220 .output()
12221 .await
12222 .context("spawn gh")?;
12223 if out.status.success() {
12224 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12225 } else {
12226 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12227 }
12228}
12229
12230async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12232 match std::fs::symlink_metadata(path) {
12233 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12234 git::worktree_remove_clean(repo, path).await?;
12237 return Ok(());
12238 }
12239 Err(e) => return Err(e.into()),
12240 Ok(_) => {}
12241 }
12242 if !git::is_clean(path).await? {
12243 bail!(
12244 "worktree has uncommitted work; refusing removal: {}",
12245 path.display()
12246 );
12247 }
12248 if !git::worktree_remove_clean(repo, path).await? {
12249 bail!("worktree removal refused: {}", path.display());
12250 }
12251 Ok(())
12252}
12253
12254pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12263 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12264 bail!("refusing fold: {reason}");
12265 }
12266 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12267 bail!("candidate preservation incomplete; refusing cleanup");
12268 }
12269 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12273 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12274 let _driver = crate::driver::maintenance(home, &state.id)?;
12275 let mut output_guards = Vec::new();
12276 for entry in std::fs::read_dir(state.worktree_root())
12277 .into_iter()
12278 .flatten()
12279 .flatten()
12280 {
12281 if entry.path().join(".git").is_file() {
12282 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12283 }
12284 }
12285 let repo = state.repo.clone();
12286 let wait = repository_wait(state, None);
12289 let _coordination = match coordinate_operation(state, &repo, &wait).await {
12290 Ok(lease) => lease,
12291 Err(e) => {
12292 state.save_under(home)?;
12293 return Err(e);
12294 }
12295 };
12296 let _source_outputs = if state.status == RunStatus::Merged
12299 && state.merge.as_ref().is_some_and(|merge| merge.ok)
12300 {
12301 verify_merged_fold_sources(state, drop_winner, home).await?
12302 } else {
12303 Vec::new()
12304 };
12305 let root = state.worktree_root();
12306 let winner = state.tally.as_ref().map(|t| t.winner);
12307 let mut removed = Vec::new();
12308
12309 for i in 0..state.candidates.len() {
12310 let c = state.candidates[i].clone();
12311 let is_winner = Some(c.label) == winner;
12312 if is_winner && !drop_winner {
12313 continue;
12314 }
12315 let existed = c.worktree.exists();
12316 remove_preserved_worktree(&repo, &c.worktree).await?;
12319 if existed {
12320 removed.push(c.worktree.to_string_lossy().into_owned());
12321 }
12322 let handed_over = state.released_branches.contains(&c.branch);
12325 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12326 if !git::branch_delete(&repo, &c.branch).await? {
12327 bail!("candidate branch removal refused");
12328 }
12329 removed.push(c.branch.clone());
12330 }
12331 state.candidates[i].folded = true;
12332 }
12333
12334 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12335 let path = name.path();
12336 let keep = !drop_winner
12337 && winner.is_some_and(|w| {
12338 path.file_name()
12339 .is_some_and(|n| n == format!("cand-{w}").as_str())
12340 });
12341 if keep {
12342 continue;
12343 }
12344 remove_preserved_worktree(&repo, &path).await?;
12345 removed.push(path.to_string_lossy().into_owned());
12346 }
12347
12348 remove_if_empty(&root);
12357
12358 if state.enabled_worktree_config && drop_winner {
12359 git::release_worktree_config(&repo).await.ok();
12363 state.enabled_worktree_config = false;
12364 }
12365 state.save_under(home)?;
12366 Ok(removed)
12367}
12368
12369async fn verify_merged_fold_sources(
12370 state: &RunState,
12371 drop_winner: bool,
12372 home: &Path,
12373) -> Result<Vec<std::fs::File>> {
12374 let winner = state.tally.as_ref().map(|t| t.winner);
12375 let mut heads = BTreeSet::new();
12376 for candidate in &state.candidates {
12377 if Some(candidate.label) == winner && !drop_winner {
12378 continue;
12379 }
12380 anyhow::ensure!(
12381 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12382 "candidate recovery source is outside the inventoried run root: {}",
12383 candidate.worktree.display()
12384 );
12385 match std::fs::symlink_metadata(&candidate.worktree) {
12386 Ok(_) => {}
12387 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12388 Err(error) => return Err(error.into()),
12389 }
12390 if !state.released_branches.contains(&candidate.branch)
12391 && git::branch_exists(&state.repo, &candidate.branch).await?
12392 {
12393 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12394 }
12395 }
12396 let mut cargo_guards = Vec::new();
12397 let entries = match std::fs::read_dir(state.worktree_root()) {
12398 Ok(entries) => Some(entries),
12399 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12400 Err(error) => return Err(error.into()),
12401 };
12402 for entry in entries.into_iter().flatten() {
12403 let path = entry?.path();
12404 let metadata = std::fs::symlink_metadata(&path)?;
12405 anyhow::ensure!(
12406 metadata.is_dir() && !metadata.file_type().is_symlink(),
12407 "non-directory or symlink recovery source retained: {}",
12408 path.display()
12409 );
12410 if !drop_winner
12411 && winner.is_some_and(|w| {
12412 path.file_name()
12413 .is_some_and(|name| name == format!("cand-{w}").as_str())
12414 })
12415 {
12416 continue;
12417 }
12418 anyhow::ensure!(
12419 path.join(".git").is_file(),
12420 "unregistered recovery source retained: {}",
12421 path.display()
12422 );
12423 anyhow::ensure!(
12424 git::git(
12425 &path,
12426 &[
12427 "status",
12428 "--porcelain",
12429 "--untracked-files=all",
12430 "--ignored",
12431 "--",
12432 ".",
12433 ":(exclude)target"
12434 ]
12435 )
12436 .await?
12437 .is_empty(),
12438 "source changes or ignored recovery files retained: {}",
12439 path.display()
12440 );
12441 match std::fs::symlink_metadata(path.join("target")) {
12442 Ok(_) => {
12443 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12444 }
12445 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12446 Err(error) => return Err(error.into()),
12447 }
12448 for checkpoint in [
12449 "rebase-merge",
12450 "rebase-apply",
12451 "sequencer",
12452 "CHERRY_PICK_HEAD",
12453 "MERGE_HEAD",
12454 "REBASE_HEAD",
12455 "REVERT_HEAD",
12456 "BISECT_LOG",
12457 "AUTO_MERGE",
12458 "MERGE_MSG",
12459 ] {
12460 let location = git::git(
12461 &path,
12462 &[
12463 "rev-parse",
12464 "--path-format=absolute",
12465 "--git-path",
12466 checkpoint,
12467 ],
12468 )
12469 .await?;
12470 anyhow::ensure!(
12471 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12472 if e.kind() == std::io::ErrorKind::NotFound),
12473 "Git recovery checkpoint retained: {checkpoint}"
12474 );
12475 }
12476 heads.insert(git::rev_parse(&path, "HEAD").await?);
12477 }
12478 crate::fold_evidence::preserve(state, home, &heads).await?;
12479
12480 Ok(cargo_guards)
12481}
12482
12483fn remove_if_empty(dir: &Path) {
12494 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12495 std::fs::remove_dir(dir).ok();
12496 }
12497}
12498
12499pub fn worst_open(state: &RunState) -> Option<Severity> {
12501 state
12502 .reviews
12503 .last()?
12504 .reviews
12505 .iter()
12506 .flat_map(|r| r.findings.iter())
12507 .map(|f| f.severity)
12508 .max()
12509}
12510
12511async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12515 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12516 git::acquire_worktree_config(repo).await?;
12517 state.enabled_worktree_config = true;
12518 }
12519 Ok(())
12520}
12521
12522#[cfg(test)]
12523mod tests {
12524 #[tokio::test]
12525 async fn launcher_base_fetch_requires_remote_success() {
12526 let tmp = tempfile::tempdir().unwrap();
12527 let repo = tmp.path().join("repo");
12528 let remote = tmp.path().join("remote");
12529 std::fs::create_dir(&repo).unwrap();
12530 std::fs::create_dir(&remote).unwrap();
12531 git::git(&remote, &["init", "--bare", "-b", "main"])
12532 .await
12533 .unwrap();
12534 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12535 git::git(
12536 &repo,
12537 &[
12538 "-c",
12539 "user.name=Fixture",
12540 "-c",
12541 "user.email=fixture@example.invalid",
12542 "commit",
12543 "--allow-empty",
12544 "-m",
12545 "fixture",
12546 ],
12547 )
12548 .await
12549 .unwrap();
12550 git::git(
12551 &repo,
12552 &["remote", "add", "origin", remote.to_str().unwrap()],
12553 )
12554 .await
12555 .unwrap();
12556 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12557 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12558 assert_eq!(
12559 resolve_base(&repo, "main", "origin", None)
12560 .await
12561 .unwrap()
12562 .commit,
12563 head
12564 );
12565 #[cfg(unix)]
12566 {
12567 let policy: crate::config::Coordination = toml::from_str(
12568 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12569 ).unwrap();
12570 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12571 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12572 let held = crate::repo_lock::SeatLease::operation_with(
12573 &repo,
12574 Some(&policy),
12575 &crate::repo_lock::WaitPolicy::default(),
12576 )
12577 .await
12578 .unwrap();
12579 assert!(held.is_some());
12580 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12581 tokio::pin!(fetch);
12582 assert!(
12583 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12584 .await
12585 .is_err()
12586 );
12587 assert!(
12588 !repo.join(".git/FETCH_HEAD").exists(),
12589 "no fetch before admission"
12590 );
12591 drop(held);
12592 let fetched = fetch.await.unwrap();
12593 assert_eq!(fetched.commit, head);
12594 assert_eq!(fetched.remote, "origin");
12595 assert_eq!(fetched.branch, "main");
12596 }
12597 git::git(
12599 &repo,
12600 &[
12601 "remote",
12602 "set-url",
12603 "origin",
12604 tmp.path().join("missing").to_str().unwrap(),
12605 ],
12606 )
12607 .await
12608 .unwrap();
12609 let error = resolve_base(&repo, "main", "origin", None)
12610 .await
12611 .unwrap_err()
12612 .to_string();
12613 assert!(error.contains("cannot read origin/main"));
12614 assert!(error.contains("refusing to branch off"));
12615 }
12616
12617 #[test]
12618 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12619 let leaked = vec!["chore(deps): update a crate".to_owned()];
12620 let own = "fix(daemon): apply a chosen action";
12621 assert!(should_retitle("", own, &leaked));
12622 assert!(should_retitle(
12623 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12624 own,
12625 &leaked
12626 ));
12627 assert!(should_retitle(
12628 "chore: land candidate A of run 1",
12629 own,
12630 &leaked
12631 ));
12632 assert!(should_retitle(
12633 "magi: candidate A (uncommitted work)",
12634 own,
12635 &leaked
12636 ));
12637 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12638 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12639 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12640 }
12641
12642 #[test]
12643 fn pr_merge_plan_creates_adopts_or_stops() {
12644 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12645 assert_eq!(
12646 pr_merge_plan(Ok(land::OpenPr::One {
12647 url: "u".into(),
12648 title: "t".into()
12649 })),
12650 PrPlan::Adopt {
12651 url: "u".into(),
12652 title: "t".into()
12653 }
12654 );
12655 let PrPlan::Stop(many) =
12656 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12657 else {
12658 panic!("many must stop");
12659 };
12660 assert!(many.contains('a') && many.contains('b'));
12661 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12662 panic!("a failed lookup must stop");
12663 };
12664 assert!(err.contains("bad token"));
12665 }
12666
12667 use super::*;
12668 use crate::run::GateStatus;
12669 use std::collections::BTreeMap;
12670 use std::time::Duration;
12671
12672 fn conductor() -> AgentSpec {
12673 AgentSpec {
12674 id: "conductor".to_owned(),
12675 kind: crate::config::AgentKind::Command,
12676 model: None,
12677 command: vec!["true".to_owned()],
12678 extra_args: Vec::new(),
12679 env: BTreeMap::new(),
12680 prompt_delivery: None,
12681 max_turn_secs: None,
12682 }
12683 }
12684
12685 fn spec(id: &str) -> AgentSpec {
12686 AgentSpec {
12687 id: id.to_owned(),
12688 kind: crate::config::AgentKind::Command,
12689 model: None,
12690 command: vec!["true".to_owned()],
12691 extra_args: Vec::new(),
12692 env: BTreeMap::new(),
12693 prompt_delivery: None,
12694 max_turn_secs: None,
12695 }
12696 }
12697
12698 fn ids(xs: &[&str]) -> BTreeSet<String> {
12699 xs.iter().map(|s| (*s).to_owned()).collect()
12700 }
12701
12702 #[test]
12703 fn pick_successor_skips_an_agent_another_seat_holds() {
12704 let roster = [spec("a"), spec("b"), spec("c")];
12706 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12707 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12708 }
12709
12710 #[test]
12711 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12712 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12714 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12715 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12716 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12717 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12718 }
12719
12720 #[test]
12721 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12722 let roster = [spec("a"), spec("b")];
12723 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12724 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12725 let carried = ids(&["b"]);
12726 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12727 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12728 }
12729
12730 #[test]
12731 fn pick_successor_returns_none_without_any_untried_successor() {
12732 let roster = [spec("a"), spec("b")];
12733 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12734 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12735 let carried = ids(&["a"]);
12736 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12737 }
12738
12739 #[test]
12740 fn pick_successor_rescue_avoids_another_seats_occupant() {
12741 let roster = [spec("a"), spec("b"), spec("c")];
12743 let carried = ids(&["b", "c"]);
12744 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12745 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12746 }
12747
12748 #[test]
12749 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12750 let roster = [spec("a"), spec("b"), spec("c")];
12751 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12752 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12753 }
12754
12755 #[test]
12756 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12757 let roster = [spec("a"), spec("b"), spec("c")];
12758 let failed = ids(&["a", "b", "c"]);
12759 let mut tried = ids(&["b"]);
12761 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12762 assert_eq!(first.id, "a");
12763 tried.insert(first.id.clone());
12764 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12765 assert_eq!(second.id, "c");
12766 tried.insert(second.id.clone());
12767 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12768 }
12769
12770 #[test]
12771 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12772 let roster = [spec("a"), spec("b")];
12773 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12774 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12775 }
12776
12777 #[test]
12778 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12779 let roster = [spec("a"), spec("b"), spec("c")];
12780 let h = SeatHistory {
12781 failed: ids(&["a"]),
12782 last_ok: Some("b".to_owned()),
12783 last_fail: None,
12784 };
12785 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12786 let gone = SeatHistory {
12788 last_ok: Some("zzz".to_owned()),
12789 ..h.clone()
12790 };
12791 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12792 let failed = SeatHistory {
12793 failed: ids(&["a", "b"]),
12794 last_ok: Some("b".to_owned()),
12795 last_fail: None,
12796 };
12797 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12798 }
12799
12800 #[test]
12801 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12802 let first = SeatState::new("review-1", "alpha", 7);
12803 let next = handover_seat("review-1", "gamma", 7);
12804 assert_ne!(first.claude_session, next.claude_session);
12805 }
12806
12807 #[test]
12808 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12809 let mut state = state_with_summary("x", "y");
12810 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12811 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12812 assert_ne!(a.claude_session, b.claude_session);
12813 }
12814
12815 #[test]
12816 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12817 let roster = [spec("a"), spec("b")];
12818 let h = SeatHistory {
12819 failed: ids(&["a", "b"]),
12820 ..SeatHistory::default()
12821 };
12822 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12823 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12824 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12825 }
12826
12827 #[test]
12834 fn failure_signature_ignores_numbers_and_paths() {
12835 assert_eq!(
12836 failure_signature("exited with Some(2) and no usable output"),
12837 failure_signature("exited with Some(137) and no usable output")
12838 );
12839 assert_eq!(
12840 failure_signature("process exited with no usable output (exit code 2)"),
12841 failure_signature("process exited with no usable output (exit code 137)")
12842 );
12843 for (old, new) in [
12844 (
12845 "exited with Some(2) and no usable output",
12846 "process exited with no usable output (exit code 2)",
12847 ),
12848 (
12849 "exited with None and no usable output",
12850 "process exited with no usable output (exit code unavailable)",
12851 ),
12852 ] {
12853 let previous = FailClass::Other(failure_signature(old));
12854 let current = FailClass::Other(failure_signature(new));
12855 assert_eq!(previous, current, "resumption retains the failure class");
12856 assert!(!should_hand_over(Some(&previous), ¤t));
12857 }
12858 assert_eq!(
12859 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12860 failure_signature("cannot open /var/x.txt: denied")
12861 );
12862 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12863 }
12864
12865 #[test]
12866 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12867 let other = FailClass::Other("x".into());
12868 assert!(should_hand_over(None, &FailClass::Quota));
12869 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12870 assert!(should_hand_over(
12871 Some(&FailClass::Timeout),
12872 &FailClass::Timeout
12873 ));
12874 assert!(should_hand_over(None, &other));
12875 assert!(!should_hand_over(Some(&other), &other));
12876 assert!(should_hand_over(
12877 Some(&other),
12878 &FailClass::Other("y".into())
12879 ));
12880 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12882 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12883 }
12884
12885 #[test]
12886 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12887 let a = SeatState::new("judge-1", "alpha", 7);
12888 let b = handover_seat("judge-1", "beta", 7);
12889 assert_ne!(a.claude_session, b.claude_session);
12890 assert_eq!(b.turns, 0);
12891 }
12892
12893 #[test]
12894 fn a_timeout_is_classified_apart_from_other_failures() {
12895 assert_eq!(
12896 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12897 Some(FailClass::Timeout)
12898 );
12899 assert!(matches!(
12900 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12901 Some(FailClass::Other(_))
12902 ));
12903 }
12904
12905 #[test]
12906 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12907 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12908 let tried = BTreeSet::from(["beta".to_owned()]);
12909 let next = next_untried_in_roster(&roster, 1, &tried);
12912 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12913 }
12914
12915 #[test]
12916 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12917 let roster = vec![spec("alpha"), spec("beta")];
12918 let tried = BTreeSet::from(["beta".to_owned()]);
12919 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12923 }
12924
12925 #[test]
12926 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12927 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12928 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12929 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12933 }
12934
12935 #[test]
12936 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12937 let roster = vec![spec("a"), spec("a"), spec("b")];
12938 let tried = BTreeSet::from(["a".to_owned()]);
12939 let next = next_untried_in_roster(&roster, 0, &tried);
12940 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12941 }
12942
12943 #[test]
12944 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12945 let roster = vec![spec("a"), spec("b")];
12946 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12947 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12948 }
12949
12950 #[test]
12951 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12952 let dir = tempfile::tempdir().unwrap();
12953 let bay = dir.path().join("ffff");
12954
12955 remove_if_empty(&bay);
12957 assert!(!bay.exists());
12958
12959 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12962 remove_if_empty(&bay);
12963 assert!(bay.exists(), "non-empty directory must survive");
12964
12965 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12967 remove_if_empty(&bay);
12968 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12969 }
12970
12971 #[test]
12980 fn a_full_panel_that_found_nothing_is_clean() {
12981 assert!(round_is_clean(
12982 0,
12983 true,
12984 2,
12985 2,
12986 0,
12987 IncompleteReviewPolicy::Block
12988 ));
12989 }
12990
12991 #[test]
12992 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12993 assert!(!round_is_clean(
12994 0,
12995 true,
12996 1,
12997 2,
12998 0,
12999 IncompleteReviewPolicy::Block
13000 ));
13001 }
13002
13003 #[test]
13004 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13005 assert!(!round_is_clean(
13006 1,
13007 true,
13008 1,
13009 2,
13010 0,
13011 IncompleteReviewPolicy::Warn
13012 ));
13013 }
13014
13015 #[test]
13016 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13017 assert!(round_is_clean(
13018 0,
13019 true,
13020 1,
13021 2,
13022 0,
13023 IncompleteReviewPolicy::Warn
13024 ));
13025 }
13026
13027 #[test]
13028 fn a_full_panel_with_an_open_finding_is_not_clean() {
13029 assert!(!round_is_clean(
13030 1,
13031 true,
13032 2,
13033 2,
13034 0,
13035 IncompleteReviewPolicy::Block
13036 ));
13037 }
13038
13039 #[test]
13040 fn a_full_panel_with_a_red_e2e_is_not_clean() {
13041 assert!(!round_is_clean(
13042 0,
13043 false,
13044 2,
13045 2,
13046 0,
13047 IncompleteReviewPolicy::Block
13048 ));
13049 }
13050
13051 #[test]
13058 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13059 assert!(round_is_clean(
13062 0,
13063 true,
13064 1,
13065 2,
13066 1,
13067 IncompleteReviewPolicy::Block
13068 ));
13069 }
13070
13071 #[test]
13072 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13073 assert!(!round_is_clean(
13076 0,
13077 true,
13078 1,
13079 2,
13080 0,
13081 IncompleteReviewPolicy::Block
13082 ));
13083 }
13084
13085 #[test]
13086 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13087 assert!(!round_is_clean(
13088 1,
13089 true,
13090 1,
13091 2,
13092 1,
13093 IncompleteReviewPolicy::Block
13094 ));
13095 assert!(!round_is_clean(
13096 0,
13097 false,
13098 1,
13099 2,
13100 1,
13101 IncompleteReviewPolicy::Block
13102 ));
13103 }
13104
13105 #[test]
13106 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13107 assert!(!round_is_clean(
13111 0,
13112 true,
13113 0,
13114 2,
13115 2,
13116 IncompleteReviewPolicy::Block
13117 ));
13118 }
13119
13120 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13121 CommandOutcome {
13122 command: "test".to_owned(),
13123 code,
13124 output_tail: String::new(),
13125 duration_ms: 0,
13126 resource_blocked,
13127 }
13128 }
13129
13130 #[test]
13131 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13132 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13133 assert!(
13134 !verify_inconclusive(&[outcome(Some(1), false)]),
13135 "an ordinary failure is still evidence about the patch"
13136 );
13137 assert!(verify_inconclusive(&[outcome(None, true)]));
13138 assert!(
13139 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13140 "one inconclusive outcome taints the whole batch"
13141 );
13142 assert!(!verify_inconclusive(&[]));
13143 }
13144
13145 #[tokio::test]
13146 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13147 let calls = std::sync::atomic::AtomicUsize::new(0);
13151 let started = Instant::now();
13152 wait_for_pids_with(
13153 &[123],
13154 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13155 Duration::from_millis(5),
13156 Duration::from_secs(5),
13157 )
13158 .await;
13159 assert!(
13160 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13161 "must keep checking rather than deciding on the first answer"
13162 );
13163 assert!(
13164 started.elapsed() < Duration::from_secs(1),
13165 "must return the moment it is confirmed dead, not wait out the ceiling"
13166 );
13167 }
13168
13169 #[tokio::test]
13170 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13171 let started = Instant::now();
13172 wait_for_pids_with(
13173 &[123],
13174 |_| true, Duration::from_millis(5),
13176 Duration::from_millis(30),
13177 )
13178 .await;
13179 let elapsed = started.elapsed();
13180 assert!(
13181 elapsed >= Duration::from_millis(30),
13182 "must not give up before its own ceiling: {elapsed:?}"
13183 );
13184 assert!(
13185 elapsed < Duration::from_secs(1),
13186 "must not wait past its own ceiling either: {elapsed:?}"
13187 );
13188 }
13189
13190 #[tokio::test]
13191 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13192 let started = Instant::now();
13193 wait_for_pids_with(
13194 &[],
13195 |_| true,
13196 Duration::from_secs(5),
13197 Duration::from_secs(5),
13198 )
13199 .await;
13200 assert!(
13201 started.elapsed() < Duration::from_millis(200),
13202 "an empty pid list has nothing to confirm"
13203 );
13204 }
13205
13206 fn review_round(
13212 clean: bool,
13213 blocking: usize,
13214 answered: usize,
13215 expected: usize,
13216 progressed: bool,
13217 e2e_ok: bool,
13218 ) -> ReviewRound {
13219 ReviewRound {
13220 round: 1,
13221 head: "h".to_owned(),
13222 verified_head: None,
13223 verified_at: None,
13224 reviews: Vec::new(),
13225 e2e: vec![CommandOutcome {
13226 command: "test".to_owned(),
13227 code: Some(if e2e_ok { 0 } else { 1 }),
13228 output_tail: String::new(),
13229 duration_ms: 0,
13230 resource_blocked: false,
13231 }],
13232 verify_retried: false,
13233 e2e_deferred: false,
13234 e2e_defer_reason: None,
13235 fix: None,
13236 blocking,
13237 answered,
13238 expected,
13239 clean,
13240 progressed,
13241 vote_split: false,
13242 reconsideration: Vec::new(),
13243 verdict: None,
13244 }
13245 }
13246
13247 #[test]
13248 fn review_conclusion_is_none_when_nothing_has_run() {
13249 assert_eq!(review_conclusion(&[], 3), None);
13250 }
13251
13252 #[test]
13253 fn review_conclusion_is_none_while_rounds_remain() {
13254 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13255 assert_eq!(review_conclusion(&rounds, 3), None);
13256 }
13257
13258 #[test]
13259 fn review_conclusion_is_gating_once_a_round_is_clean() {
13260 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13261 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13262 }
13263
13264 #[test]
13265 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13266 let rounds = vec![
13267 review_round(false, 1, 2, 2, true, true),
13268 review_round(false, 1, 2, 2, true, true),
13269 ];
13270 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13271 }
13272
13273 #[test]
13274 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13275 let rounds = vec![
13276 review_round(false, 1, 2, 2, true, true),
13277 review_round(false, 1, 2, 2, true, false),
13278 ];
13279 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13280 }
13281
13282 #[test]
13283 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13284 let mut blocked = review_round(false, 1, 2, 2, true, false);
13291 blocked.e2e[0].resource_blocked = true;
13292 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13293 assert_eq!(review_conclusion(&rounds, 2), None);
13294 }
13295
13296 #[test]
13297 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13298 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13300 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13301 }
13302
13303 #[test]
13304 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13305 let rounds = vec![
13306 review_round(false, 1, 2, 2, false, true),
13307 review_round(false, 1, 2, 2, false, true),
13308 ];
13309 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13310 }
13311
13312 fn secs(n: u64) -> Duration {
13313 Duration::from_secs(n)
13314 }
13315
13316 #[tokio::test]
13319 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13320 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13321 if std::env::var_os(CHILD).is_none() {
13322 let output = std::process::Command::new(std::env::current_exe().unwrap())
13326 .args([
13327 "--exact",
13328 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13329 "--nocapture",
13330 ])
13331 .env(CHILD, "1")
13332 .env_remove("MAGI_WORKER_HOME")
13333 .env_remove("MAGI_WORKER_ID")
13334 .output()
13335 .unwrap();
13336 assert!(
13337 output.status.success()
13338 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13339 "{}\n{}",
13340 String::from_utf8_lossy(&output.stdout),
13341 String::from_utf8_lossy(&output.stderr)
13342 );
13343 return;
13344 }
13345 ask_test_home();
13346 let tmp = tempfile::tempdir().unwrap();
13347 let repo = tmp.path().join("repo");
13348 std::fs::create_dir_all(&repo).unwrap();
13349 init_repo(&repo);
13350 let g = |args: &[&str]| {
13351 let out = std::process::Command::new("git")
13352 .args(args)
13353 .current_dir(&repo)
13354 .quiet()
13355 .output()
13356 .unwrap();
13357 assert!(
13358 out.status.success(),
13359 "{}",
13360 String::from_utf8_lossy(&out.stderr)
13361 );
13362 };
13363 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13364 g(&["checkout", "-b", "historical"]);
13365 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13366 g(&["add", "."]);
13367 g(&["commit", "-m", "old alternative"]);
13368 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13369 g(&["checkout", "-b", "magi/task-selected", &base]);
13370 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13371 g(&["add", "."]);
13372 g(&["commit", "-m", "measurement infrastructure"]);
13373 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13374 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13375 g(&["add", "."]);
13376 g(&["commit", "-m", "dependent implementation"]);
13377 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13378 g(&["checkout", "main"]);
13379 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13380 g(&["add", "."]);
13381 g(&["commit", "-m", "unrelated base change"]);
13382 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13383 let origin = tmp.path().join("origin.git");
13384 g(&[
13385 "clone",
13386 "--bare",
13387 repo.to_str().unwrap(),
13388 origin.to_str().unwrap(),
13389 ]);
13390 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13391 g(&["fetch", "origin"]);
13392 let selection = crate::pr_recovery::Selection {
13393 url: "https://example.test/project/pull/1".into(),
13394 branch: "magi/task-selected".into(),
13395 head: head.clone(),
13396 source_head: None,
13397 source_run: None,
13398 source_proof: None,
13399 disposition: Some(crate::pr_recovery::Disposition::Review),
13400 instruction: "preserve measured behavior".into(),
13401 approved_at: jiff::Timestamp::now(),
13402 };
13403 let mut template = runner_at(RunStatus::Reviewing);
13404 template.state.repo = repo.clone();
13405 template.state.base_commit = current.clone();
13406 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13407 template.state.origin = Some(crate::run::Origin::queue("selected"));
13408 template.state.pr_head = Some(selection.branch.clone());
13409 template.state.pr_recovery = Some(selection.clone());
13410 let mut runner =
13411 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13412 .await
13413 .unwrap();
13414 let wt = runner.state.winner().unwrap().worktree.clone();
13415 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13416 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13417 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13418 assert!(runner.state.seeds.is_empty());
13419 assert_eq!(runner.state.instruction, "preserve measured behavior");
13420 runner.sync_to_base().await.unwrap();
13421 assert_eq!(
13422 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13423 "prerequisite"
13424 );
13425 assert_eq!(
13426 std::fs::read_to_string(wt.join("implementation")).unwrap(),
13427 "measured implementation"
13428 );
13429 assert_eq!(
13430 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13431 "current main change"
13432 );
13433 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13434 assert_eq!(
13435 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13436 head
13437 );
13438
13439 let mut partial = runner_at(RunStatus::Reviewing);
13441 partial.state.repo = repo.clone();
13442 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13443 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13444 run: Some("exit 1".into()),
13445 ..Default::default()
13446 }];
13447 let selection = runner.state.pr_recovery.clone().unwrap();
13448 partial.state.pr_recovery = Some(selection.clone());
13449 partial.state.pr_head = Some(selection.branch.clone());
13450 partial.state.origin = Some(crate::run::Origin::queue("selected"));
13451 let id = partial.state.id.clone();
13452 let error = match Runner::open_selected(
13453 &repo,
13454 partial.state,
13455 partial.roles,
13456 1,
13457 runner.state.base_commit.clone(),
13458 selection,
13459 )
13460 .await
13461 {
13462 Ok(_) => panic!("setup must fail"),
13463 Err(e) => e,
13464 };
13465 assert_eq!(
13466 error
13467 .downcast_ref::<crate::pr_recovery::StartupFailed>()
13468 .unwrap()
13469 .run,
13470 id
13471 );
13472 let failed = RunState::load(&id).unwrap();
13473 assert_eq!(failed.status, RunStatus::Blocked);
13474 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13475 assert!(Runner::resume(&id).is_err());
13476 runner.state.status = RunStatus::Ready;
13477 runner.state.driver_exited = true;
13478 runner.state.save().unwrap();
13479 let mut retry = runner_at(RunStatus::Reviewing);
13480 retry.state.repo = repo.clone();
13481 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13482 let mut reselected = runner.state.pr_recovery.clone().unwrap();
13483 reselected.approved_at = jiff::Timestamp::now();
13484 retry.state.origin = Some(crate::run::Origin::queue("selected"));
13485 retry.state.pr_head = Some(reselected.branch.clone());
13486 retry.state.pr_recovery = Some(reselected.clone());
13487 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13488 .await
13489 .unwrap();
13490 let restarted = Runner::open_selected(
13491 &repo,
13492 retry.state,
13493 retry.roles,
13494 1,
13495 runner.state.base_commit.clone(),
13496 reselected,
13497 )
13498 .await
13499 .unwrap();
13500 assert_eq!(
13501 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13502 .await
13503 .unwrap(),
13504 head
13505 );
13506 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13507 assert_eq!(
13508 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13509 head
13510 );
13511 }
13512
13513 #[tokio::test]
13514 #[cfg(unix)]
13515 async fn fallback_and_interrupted_restoration_require_coordination() {
13516 use std::os::fd::AsRawFd as _;
13517 ask_test_home();
13518 let tmp = tempfile::tempdir().unwrap();
13519 let mut runner = registered_runner(tmp.path()).await;
13520 runner.state.config.graph.timeout_resource_wait = 1;
13521 let winner = runner.state.candidates[0].clone();
13522 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13523 let leftover = winner.worktree.join("leftover");
13524 std::fs::write(&leftover, "untrusted edits").unwrap();
13525 let next = tmp.path().join("next-fixer");
13526 let mut failed = spec("first");
13527 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13528 let mut successor = spec("second");
13529 successor.command = vec![
13530 "sh".into(),
13531 "-c".into(),
13532 format!("touch '{}'; echo done", next.display()),
13533 ];
13534 let attempts = vec![
13535 (failed.clone(), "first".into()),
13536 (successor.clone(), "second".into()),
13537 ];
13538 let lock_path = runner.state.repo.join(".git/updater.lock");
13539 let external = std::fs::File::create(&lock_path).unwrap();
13540 assert_eq!(
13541 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13542 0
13543 );
13544 let artifacts = tmp.path().join("artifacts");
13545 let build = |spec, seat| SeatJob {
13546 spec,
13547 seat,
13548 cwd: winner.worktree.clone(),
13549 prompt: "fixture".into(),
13550 timeout: Duration::from_secs(10),
13551 allow_write: true,
13552 sessions: false,
13553 artifacts: artifacts.clone(),
13554 stem: "fix".into(),
13555 handover: None,
13556 };
13557 let (_, _, out) = runner
13558 .ask_fixer_among(
13559 attempts.clone(),
13560 "graft",
13561 None,
13562 Some((&winner.worktree, &head)),
13563 build,
13564 )
13565 .await;
13566 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13567 assert!(leftover.exists(), "contention must prevent clean");
13568 assert!(!next.exists());
13569 assert!(runner.state.handovers.is_empty());
13570 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13572 from_run: "earlier".into(),
13573 generation: 1,
13574 mode: crate::recovery::Mode::VerifyRepair,
13575 reasons: Vec::new(),
13576 reviewed_head: None,
13577 rounds: Vec::new(),
13578 open_findings: Vec::new(),
13579 evidence: None,
13580 criteria_inherited: true,
13581 repair: crate::recovery::Repair::Started {
13582 before: head.clone(),
13583 },
13584 });
13585 assert!(
13586 runner
13587 .recovery_repair(&winner)
13588 .await
13589 .unwrap_err()
13590 .is::<crate::repo_lock::ResourceBlocked>()
13591 );
13592 assert!(matches!(
13593 runner.state.recovery.as_ref().unwrap().repair,
13594 crate::recovery::Repair::Started { .. }
13595 ));
13596 assert!(leftover.exists());
13597 drop(external);
13598 runner.recovery_repair(&winner).await.unwrap();
13599 assert!(!leftover.exists());
13600 assert!(matches!(
13601 runner.state.recovery.as_ref().unwrap().repair,
13602 crate::recovery::Repair::Done { .. }
13603 ));
13604 std::fs::remove_file(&lock_path).unwrap();
13606 std::fs::create_dir(&lock_path).unwrap();
13607 std::fs::write(&leftover, "untrusted edits").unwrap();
13608 let (_, _, out) = runner
13609 .ask_fixer_among(
13610 attempts,
13611 "graft",
13612 None,
13613 Some((&winner.worktree, &head)),
13614 build,
13615 )
13616 .await;
13617 assert!(matches!(out, AgentOutcome::Failed(_)));
13618 assert!(leftover.exists());
13619 assert!(!next.exists());
13620 assert_eq!(
13621 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13622 head
13623 );
13624 }
13625
13626 #[cfg(unix)]
13627 async fn registered_runner(root: &Path) -> Runner {
13628 ask_test_home();
13629 let repo = root.join("repo");
13630 std::fs::create_dir_all(&repo).unwrap();
13631 init_repo(&repo);
13632 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13633 let mut runner = runner_at(RunStatus::Prep);
13634 runner.state.repo = repo.clone();
13635 runner.state.config.coordination = crate::config::test_coordination();
13636 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13637 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13638 runner.state.config.graph.advise = false;
13639 runner.roles.implementers = vec![spec("implement")];
13640 runner.roles.judges = vec![spec("judge")];
13641 runner.prep().await.unwrap();
13642 runner
13643 }
13644
13645 #[tokio::test]
13646 #[cfg(unix)]
13647 async fn parked_implementer_persists_completed_work() {
13648 for registered in [false, true] {
13649 let tmp = tempfile::tempdir().unwrap();
13650 let mut runner = registered_runner(tmp.path()).await;
13651 if !registered {
13652 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13653 }
13654 let marker = tmp.path().join("started");
13655 let release = tmp.path().join("release");
13656 let mut implementer = spec("implement");
13657 implementer.command = vec![
13658 "sh".into(),
13659 "-c".into(),
13660 format!(
13661 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13662 marker.display(),
13663 release.display()
13664 ),
13665 ];
13666 runner.roles.implementers = vec![implementer.clone()];
13667 runner.state.config.agents = vec![implementer];
13668 let pause = runner.pause.clone();
13669 let (result, ()) = tokio::join!(runner.implement(), async {
13670 tokio::time::timeout(Duration::from_secs(10), async {
13671 while !marker.exists() {
13672 tokio::time::sleep(Duration::from_millis(10)).await;
13673 }
13674 })
13675 .await
13676 .unwrap();
13677 pause.park();
13678 std::fs::write(&release, "go").unwrap();
13679 });
13680 result.unwrap();
13681 let candidate = &runner.state.candidates[0];
13682 assert!(candidate.commits > 0, "{candidate:?}");
13683 assert!(candidate.summary.contains("saved-result"));
13684 assert_eq!(candidate.files, 1);
13685 let before = candidate.commits;
13686 runner.implement().await.unwrap();
13687 assert_eq!(runner.state.candidates[0].commits, before);
13688 }
13689 }
13690
13691 #[tokio::test]
13692 #[cfg(unix)]
13693 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13694 let tmp = tempfile::tempdir().unwrap();
13695 let mut runner = registered_runner(tmp.path()).await;
13696 let winner = runner.state.candidates[0].clone();
13697 let cache = tmp.path().join("cache");
13698 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13699 runner.state.config.graph.timeout_resource_wait = 0;
13700 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13701 let crate::cache::AcquireOutcome::Acquired(held) =
13702 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13703 else {
13704 panic!("cache must be free")
13705 };
13706 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13707 from_run: "earlier".into(),
13708 generation: 1,
13709 mode: crate::recovery::Mode::VerifyRepair,
13710 reasons: Vec::new(),
13711 reviewed_head: None,
13712 rounds: Vec::new(),
13713 open_findings: Vec::new(),
13714 evidence: None,
13715 criteria_inherited: true,
13716 repair: crate::recovery::Repair::NotStarted,
13717 });
13718 runner.recovery_repair(&winner).await.unwrap();
13719 assert!(matches!(
13720 runner.state.recovery.as_ref().unwrap().repair,
13721 crate::recovery::Repair::WaitingForRestore { .. }
13722 ));
13723 drop(held);
13724 runner.state.resource_block = None;
13725 runner.state.config.graph.timeout_resource_wait = 1;
13726 runner.recovery_repair(&winner).await.unwrap();
13727 assert!(matches!(
13728 runner.state.recovery.as_ref().unwrap().repair,
13729 crate::recovery::Repair::Done { .. }
13730 ));
13731 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13732 }
13733
13734 #[tokio::test]
13735 #[cfg(unix)]
13736 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13737 use std::os::fd::AsRawFd as _;
13738 for graft in [false, true] {
13739 let tmp = tempfile::tempdir().unwrap();
13740 let mut runner = registered_runner(tmp.path()).await;
13741 runner.state.config.graph.timeout_resource_wait = 1;
13742 let winner = runner.state.candidates[0].clone();
13743 let marker = tmp.path().join("fixer-started");
13744 let mut fixer = spec("fixer");
13745 fixer.command = vec![
13746 "sh".into(),
13747 "-c".into(),
13748 format!("echo started >> '{}'; echo done", marker.display()),
13749 ];
13750 runner.roles.fixer = Some(vec![fixer]);
13751 if graft {
13752 let mut loser = winner.clone();
13753 loser.label = 'B';
13754 loser.index = 1;
13755 runner.state.candidates.push(loser);
13756 runner.state.config.graph.graft_runner_up = true;
13757 runner.state.tally = Some(Tally {
13758 first_choice: BTreeMap::new(),
13759 borda: BTreeMap::new(),
13760 winner: winner.label,
13761 rankings: 1,
13762 unanimous_initial: true,
13763 deliberated: false,
13764 changed_votes: 0,
13765 unanimous_final: true,
13766 tie_break: None,
13767 judges: 1,
13768 present: 1,
13769 quorum: 1,
13770 met_quorum: true,
13771 uncontested: None,
13772 });
13773 runner.state.judgements.push(crate::run::Judgement {
13774 judge: 1,
13775 seat: "judge-1".into(),
13776 agent: "judge".into(),
13777 ranking: vec![winner.label, 'B'],
13778 reasons: BTreeMap::new(),
13779 confidence: None,
13780 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13781 order: vec![0, 1],
13782 failed: None,
13783 duration_ms: 0,
13784 });
13785 } else {
13786 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13787 from_run: "earlier".into(),
13788 generation: 1,
13789 mode: crate::recovery::Mode::VerifyRepair,
13790 reasons: Vec::new(),
13791 reviewed_head: None,
13792 rounds: Vec::new(),
13793 open_findings: Vec::new(),
13794 evidence: None,
13795 criteria_inherited: true,
13796 repair: crate::recovery::Repair::NotStarted,
13797 });
13798 }
13799 let reader = crate::repo_lock::SeatLease::for_checkout(
13800 &winner.worktree,
13801 false,
13802 crate::config::test_coordination().as_ref(),
13803 )
13804 .await
13805 .unwrap();
13806 let pause = runner.pause.clone();
13807 let (result, ()) = tokio::join!(
13808 async {
13809 if graft {
13810 runner.graft().await
13811 } else {
13812 runner.recovery_repair(&winner).await
13813 }
13814 },
13815 async {
13816 tokio::time::sleep(Duration::from_millis(100)).await;
13817 pause.park();
13818 }
13819 );
13820 result.unwrap();
13821 assert!(!marker.exists(), "a blocked seat must not spawn");
13822 runner.state = RunState::load(&runner.state.id).unwrap();
13824 if graft {
13825 assert_eq!(
13826 runner.state.graft.as_ref().unwrap().outcome,
13827 GraftOutcome::WaitingForRestore
13828 );
13829 } else {
13830 assert!(matches!(
13831 runner.state.recovery.as_ref().unwrap().repair,
13832 crate::recovery::Repair::WaitingForRestore { .. }
13833 ));
13834 }
13835 drop(reader);
13836 runner.pause = Pause::new();
13837 runner.state.resource_block = None;
13838 let external =
13840 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13841 assert_eq!(
13842 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13843 0
13844 );
13845 let leftover = winner.worktree.join("leftover");
13846 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13847 let result = if graft {
13848 runner.graft().await
13849 } else {
13850 runner.recovery_repair(&winner).await
13851 };
13852 assert!(
13853 result
13854 .unwrap_err()
13855 .is::<crate::repo_lock::ResourceBlocked>()
13856 );
13857 assert!(leftover.exists());
13858 assert!(!marker.exists());
13859 drop(external);
13860 runner.state.resource_block = None;
13861 let result = if graft {
13862 runner.graft().await
13863 } else {
13864 runner.recovery_repair(&winner).await
13865 };
13866 result.unwrap();
13867 assert!(!leftover.exists(), "restore must precede retry");
13868 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13869 if graft {
13870 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13871 } else {
13872 assert!(matches!(
13873 runner.state.recovery.as_ref().unwrap().repair,
13874 crate::recovery::Repair::Done { .. }
13875 ));
13876 }
13877 }
13878 }
13879
13880 #[tokio::test]
13881 #[cfg(unix)]
13882 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13883 use std::os::fd::AsRawFd as _;
13884 let tmp = tempfile::tempdir().unwrap();
13885 let mut runner = registered_runner(tmp.path()).await;
13886 runner.state.config.graph.timeout_resource_wait = 1;
13887 let winner = runner.state.candidates[0].clone();
13888 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13889 runner.state.tally = Some(Tally {
13890 first_choice: BTreeMap::new(),
13891 borda: BTreeMap::new(),
13892 winner: winner.label,
13893 rankings: 0,
13894 unanimous_initial: false,
13895 deliberated: false,
13896 changed_votes: 0,
13897 unanimous_final: false,
13898 tie_break: None,
13899 judges: 0,
13900 present: 0,
13901 quorum: 0,
13902 met_quorum: true,
13903 uncontested: Some("fixture".into()),
13904 });
13905 runner.state.graft = Some(GraftRecord {
13906 pre: pre.clone(),
13907 post: None,
13908 lines: None,
13909 limit: None,
13910 agent: None,
13911 outcome: GraftOutcome::Running,
13912 requested: Vec::new(),
13913 adoption: None,
13914 adoption_error: None,
13915 });
13916 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13917 git::commit_all(&winner.worktree, "unfinished graft")
13918 .await
13919 .unwrap();
13920 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13921 let leftover = winner.worktree.join("leftover");
13922 std::fs::write(&leftover, "untrusted edits").unwrap();
13923 let path = runner.state.repo.join(".git/updater.lock");
13924 let external = std::fs::File::create(&path).unwrap();
13925 assert_eq!(
13926 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13927 0
13928 );
13929 assert!(
13930 runner
13931 .graft()
13932 .await
13933 .unwrap_err()
13934 .is::<crate::repo_lock::ResourceBlocked>()
13935 );
13936 assert!(matches!(
13937 runner.state.graft.as_ref().unwrap().outcome,
13938 GraftOutcome::Running
13939 ));
13940 assert_eq!(
13941 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13942 unfinished
13943 );
13944 assert!(leftover.exists());
13945 drop(external);
13946 std::fs::remove_file(&path).unwrap();
13947 std::fs::create_dir(&path).unwrap();
13948 assert!(runner.graft().await.is_err());
13949 assert!(matches!(
13950 runner.state.graft.as_ref().unwrap().outcome,
13951 GraftOutcome::Running
13952 ));
13953 assert_eq!(
13954 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13955 unfinished
13956 );
13957 assert!(leftover.exists());
13958 std::fs::remove_dir(&path).unwrap();
13959 runner.graft().await.unwrap();
13960 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13961 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13962 assert!(!leftover.exists());
13963 }
13964
13965 #[tokio::test]
13966 #[cfg(unix)]
13967 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13968 use std::os::fd::AsRawFd as _;
13969 let tmp = tempfile::tempdir().unwrap();
13970 let mut runner = registered_runner(tmp.path()).await;
13971 runner.state.prep_complete = Some(false);
13972 runner.state.config.graph.timeout_resource_wait = 1;
13973 runner.state.save().unwrap();
13974 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13975 assert_eq!(
13976 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13977 0
13978 );
13979 runner.execute().await.unwrap();
13980 let disk = RunState::load(&runner.state.id).unwrap();
13981 assert!(disk.driver_exited);
13982 assert_eq!(disk.driver_pid, None);
13983 assert_eq!(runner.state.status, RunStatus::Blocked);
13984 assert_eq!(
13985 runner.state.resource_block.as_ref().unwrap().kind,
13986 "repository coordination"
13987 );
13988 assert!(runner.state.prep_failure.is_none());
13989 assert!(runner.state.seats.is_empty());
13990 runner.pause.park();
13991 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13994 .await
13995 .unwrap()
13996 .unwrap();
13997 assert!(runner.state.parked);
13998 assert!(runner.state.prep_failure.is_none());
13999 let disk = RunState::load(&runner.state.id).unwrap();
14000 assert!(disk.driver_exited);
14001 assert_eq!(disk.driver_pid, None);
14002 }
14003
14004 #[tokio::test]
14005 #[cfg(unix)]
14006 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14007 use std::os::fd::AsRawFd as _;
14008 let tmp = tempfile::tempdir().unwrap();
14009 let mut runner = registered_runner(tmp.path()).await;
14010 runner.state.prep_complete = Some(false);
14011 runner.state.config.graph.timeout_resource_wait = 30;
14012 runner.state.save().unwrap();
14013 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14014 assert_eq!(
14015 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14016 0
14017 );
14018 assert!(
14019 tokio::time::timeout(Duration::from_millis(100), runner.execute())
14020 .await
14021 .is_err()
14022 );
14023 let disk = RunState::load(&runner.state.id).unwrap();
14024 assert!(disk.driver_exited);
14025 assert_eq!(disk.driver_pid, None);
14026 assert!(disk.status.resumable());
14027 assert!(runner.state.driver_exited);
14028 assert_eq!(runner.state.driver_pid, None);
14029 runner.state.save().unwrap();
14030 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14031 }
14032
14033 #[tokio::test]
14034 #[cfg(unix)]
14035 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14036 let tmp = tempfile::tempdir().unwrap();
14037 let mut runner = registered_runner(tmp.path()).await;
14038 let winner = runner.state.candidates[0].clone();
14039 let cache = tmp.path().join("cache");
14040 let started = tmp.path().join("pre-gate-started");
14041 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14042 runner.state.config.verify.gate =
14043 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14044 let home = crate::run::home();
14045 let owner =
14046 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14047 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14048 crate::cache::AcquireOutcome::Acquired(g) => g,
14049 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14050 };
14051 let mut gate = Box::pin(runner.run_gate(&winner));
14052 assert!(
14053 tokio::time::timeout(Duration::from_millis(150), &mut gate)
14054 .await
14055 .is_err()
14056 );
14057 assert!(!started.exists(), "pre-gate must wait for the cache too");
14058 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14062 let editor = crate::repo_lock::SeatLease::checkout_with(
14063 &winner.worktree,
14064 true,
14065 crate::config::test_coordination().as_ref(),
14066 &wait,
14067 )
14068 .await
14069 .unwrap();
14070 drop(editor);
14071 guard.release();
14072 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14073 .await
14074 .unwrap()
14075 .unwrap();
14076 drop(gate);
14077 assert!(outcomes.iter().all(CommandOutcome::ok));
14078 assert!(started.exists());
14079 }
14080
14081 #[tokio::test]
14082 #[cfg(unix)]
14083 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14084 use std::os::fd::AsRawFd as _;
14085
14086 ask_test_home();
14087 let tmp = tempfile::tempdir().unwrap();
14088 let repo = tmp.path().join("repo");
14089 std::fs::create_dir_all(&repo).unwrap();
14090 init_repo(&repo);
14091 let common = repo.join(".git");
14092 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14093 let mut runner = runner_at(RunStatus::Prep);
14094 runner.state.repo = repo.clone();
14095 runner.state.config.coordination = crate::config::test_coordination();
14096 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14097 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14098 runner.state.config.graph.advise = false;
14099 runner.roles.implementers = vec![spec("implement")];
14100 runner.roles.judges = vec![spec("judge")];
14101 runner.prep().await.unwrap();
14102 let winner = runner.state.candidates[0].clone();
14103 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14106 let completed = tmp.path().join("seat-completed");
14107 runner.roles.implementers[0].command = vec![
14108 "sh".into(),
14109 "-c".into(),
14110 format!("touch '{}'; echo completed", completed.display()),
14111 ];
14112 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14113 assert_eq!(
14114 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14115 0
14116 );
14117 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14118 let mut implement = Box::pin(runner.implement());
14119 tokio::select! {
14120 result = &mut implement => panic!("rescue must wait: {result:?}"),
14121 result = tokio::time::timeout(Duration::from_secs(10), async {
14122 while !completed.exists() {
14123 tokio::time::sleep(Duration::from_millis(20)).await;
14124 }
14125 }) => result.unwrap(),
14126 }
14127 assert!(
14128 tokio::time::timeout(Duration::from_millis(200), &mut implement)
14129 .await
14130 .is_err()
14131 );
14132 assert_eq!(
14133 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14134 before
14135 );
14136 assert_eq!(
14137 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14138 0
14139 );
14140 tokio::time::timeout(Duration::from_secs(10), &mut implement)
14141 .await
14142 .unwrap()
14143 .unwrap();
14144 drop(implement);
14145 assert_ne!(
14146 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14147 before
14148 );
14149 assert!(git::is_clean(&winner.worktree).await.unwrap());
14150
14151 let started = tmp.path().join("pre-gate-started");
14152 let proceed = tmp.path().join("pre-gate-proceed");
14153 let checked = tmp.path().join("gate-ran");
14154 runner.state.config.verify.pre_gate = vec![format!(
14155 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14156 started.display(),
14157 proceed.display()
14158 )];
14159 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14160 assert_eq!(
14161 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14162 0
14163 );
14164 let mut gate = Box::pin(runner.run_gate(&winner));
14165 assert!(
14166 tokio::time::timeout(Duration::from_millis(200), &mut gate)
14167 .await
14168 .is_err()
14169 );
14170 assert!(!started.exists());
14171 assert!(!checked.exists());
14172 assert!(git::is_clean(&winner.worktree).await.unwrap());
14173 assert_eq!(
14174 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14175 0
14176 );
14177 tokio::select! {
14178 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14179 result = tokio::time::timeout(Duration::from_secs(10), async {
14180 while !started.exists() {
14181 tokio::time::sleep(Duration::from_millis(20)).await;
14182 }
14183 }) => result.unwrap(),
14184 }
14185 assert_eq!(
14186 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14187 -1,
14188 "mechanical edits must hold the real lock"
14189 );
14190 std::fs::write(proceed, "ready").unwrap();
14191 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14192 .await
14193 .unwrap()
14194 .unwrap();
14195 drop(gate);
14196 assert!(outcomes.iter().all(CommandOutcome::ok));
14197 assert!(checked.exists());
14198 assert!(runner.state.pre_gate_commit.is_some());
14199 assert!(git::is_clean(&winner.worktree).await.unwrap());
14200 assert_eq!(
14201 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14202 0,
14203 "gate completion must release coordination"
14204 );
14205 drop(outsider);
14206 std::fs::remove_file(common.join("updater.lock")).unwrap();
14207 std::fs::create_dir(common.join("updater.lock")).unwrap();
14208 std::fs::remove_file(&started).unwrap();
14209 std::fs::remove_file(&checked).unwrap();
14210 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14211 let error = runner.run_gate(&winner).await.unwrap_err();
14212 assert!(
14213 error
14214 .to_string()
14215 .contains("acquire mandatory repository lock")
14216 );
14217 assert!(
14218 !started.exists(),
14219 "failed acquisition must prevent pre-gate"
14220 );
14221 assert!(!checked.exists(), "failed acquisition must prevent gate");
14222 assert!(git::is_clean(&winner.worktree).await.unwrap());
14223 assert_eq!(
14224 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14225 head
14226 );
14227 }
14228
14229 #[tokio::test]
14230 #[cfg(unix)]
14231 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14232 use std::os::fd::AsRawFd as _;
14233 use std::time::Duration;
14234
14235 ask_test_home();
14236 let tmp = tempfile::tempdir().unwrap();
14237 let repo = tmp.path().join("repo");
14238 std::fs::create_dir_all(&repo).unwrap();
14239 init_repo(&repo);
14240 let common = repo.join(".git");
14241 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14242 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14243 assert_eq!(
14244 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14245 0
14246 );
14247 let mut runner = runner_at(RunStatus::Prep);
14248 runner.state.repo = repo.clone();
14249 runner.state.config.coordination = crate::config::test_coordination();
14250 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14251 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14252 runner.state.config.graph.advise = true;
14253 runner.state.config.graph.advisors = 1;
14254 runner.roles.implementers = vec![spec("implement")];
14255 runner.roles.judges = vec![spec("judge")];
14256 let root = runner.state.worktree_root();
14257 let started = tmp.path().join("setup-started");
14258 let proceed = tmp.path().join("setup-proceed");
14259 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14260 run: Some(format!(
14261 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14262 started.display(),
14263 proceed.display()
14264 )),
14265 ..Default::default()
14266 }];
14267 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14268 let config = git::git(&repo, &["config", "--local", "--list"])
14269 .await
14270 .unwrap();
14271 let mut prep = Box::pin(runner.prep());
14272 assert!(
14273 tokio::time::timeout(Duration::from_millis(200), &mut prep)
14274 .await
14275 .is_err()
14276 );
14277 assert!(
14278 !root.exists(),
14279 "contention must prevent even checkout creation"
14280 );
14281 assert!(!started.exists(), "contention must prevent setup");
14282 assert_eq!(
14283 git::git(&repo, &["branch", "--list"]).await.unwrap(),
14284 branches
14285 );
14286 assert_eq!(
14287 git::git(&repo, &["config", "--local", "--list"])
14288 .await
14289 .unwrap(),
14290 config
14291 );
14292 drop(lock);
14293 tokio::select! {
14294 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14295 result = tokio::time::timeout(Duration::from_secs(10), async {
14296 while !started.exists() {
14297 tokio::time::sleep(Duration::from_millis(20)).await;
14298 }
14299 }) => result.unwrap(),
14300 }
14301 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14302 assert_eq!(
14303 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14304 -1
14305 );
14306 assert!(
14307 tokio::time::timeout(
14308 Duration::from_millis(100),
14309 crate::repo_lock::SeatLease::for_checkout(
14310 &repo,
14311 true,
14312 crate::config::test_coordination().as_ref()
14313 )
14314 )
14315 .await
14316 .is_err(),
14317 "preparation must also exclude local editors"
14318 );
14319 std::fs::write(proceed, "ready").unwrap();
14320 tokio::time::timeout(Duration::from_secs(10), &mut prep)
14321 .await
14322 .unwrap()
14323 .unwrap();
14324 drop(prep);
14325 assert_eq!(runner.state.prep_complete, Some(true));
14326 assert!(root.join("judge-1").exists());
14327 assert!(root.join("advisor-1").exists());
14328 assert_eq!(
14329 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14330 0
14331 );
14332 }
14333
14334 #[tokio::test]
14335 #[cfg(unix)]
14336 async fn preparation_lock_failure_prevents_repository_changes() {
14337 ask_test_home();
14338 let tmp = tempfile::tempdir().unwrap();
14339 let repo = tmp.path().join("repo");
14340 std::fs::create_dir_all(&repo).unwrap();
14341 init_repo(&repo);
14342 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14343 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14346 let mut runner = runner_at(RunStatus::Prep);
14347 runner.state.repo = repo.clone();
14348 runner.state.config.coordination = crate::config::test_coordination();
14349 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14350 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14351 runner.roles.implementers = vec![spec("implement")];
14352 let error = runner.prep().await.unwrap_err();
14353 assert!(
14354 error
14355 .to_string()
14356 .contains("acquire mandatory repository lock")
14357 );
14358 assert!(!runner.state.worktree_root().exists());
14359 assert!(runner.state.candidates.is_empty());
14360 assert!(!repo.join(".git/config.worktree").exists());
14361 assert_eq!(
14362 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14363 "* main"
14364 );
14365 }
14366
14367 #[tokio::test]
14368 async fn preparation_records_creation_intent_before_git_can_fail() {
14369 ask_test_home();
14370 let tmp = tempfile::tempdir().unwrap();
14371 let repo = tmp.path().join("repo");
14372 std::fs::create_dir_all(&repo).unwrap();
14373 init_repo(&repo);
14374 let mut runner = runner_at(RunStatus::Prep);
14375 runner.state.repo = repo.clone();
14376 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14377 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14378 runner.state.config.blind.commit_msg_hook = false;
14379 runner.roles.implementers = vec![spec("one")];
14380 let label = blind::assign_labels(1, runner.state.seed)[0];
14381 let branch = runner.state.branch_for(label);
14382 git::git(&repo, &["branch", &branch]).await.unwrap();
14385 assert!(runner.prep().await.is_err());
14386 let saved = RunState::load(&runner.state.id).unwrap();
14387 assert!(saved.candidates.is_empty());
14388 let event = saved
14389 .events
14390 .iter()
14391 .find(|e| e.node == "prep-intent")
14392 .unwrap();
14393 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14394 assert_eq!(intent["branch"], branch);
14395 assert_eq!(intent["start"], runner.state.base_commit);
14396 assert_eq!(
14397 intent["worktree"],
14398 runner
14399 .state
14400 .worktree_root()
14401 .join(format!("cand-{label}"))
14402 .to_string_lossy()
14403 .as_ref()
14404 );
14405 assert_eq!(intent["picks"], serde_json::json!([]));
14406 }
14407
14408 #[tokio::test]
14409 async fn invalid_pinned_sources_create_no_partial_worktrees() {
14410 ask_test_home();
14411 let tmp = tempfile::tempdir().unwrap();
14412 let repo = tmp.path().join("repo");
14413 std::fs::create_dir_all(&repo).unwrap();
14414 init_repo(&repo);
14415 let mut runner = runner_at(RunStatus::Prep);
14416 runner.state.repo = repo.clone();
14417 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14418 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14419 runner.state.sources.pick_commits.push(refs::Input {
14420 token: "missing".into(),
14421 sha: "0".repeat(40),
14422 provenance: refs::Provenance::ExplicitCli,
14423 });
14424 assert!(runner.prep().await.is_err());
14425 assert!(runner.state.candidates.is_empty());
14426 assert!(!runner.state.worktree_root().exists());
14427 }
14428
14429 #[tokio::test]
14430 async fn prose_and_legacy_inference_never_import_into_candidates() {
14431 ask_test_home();
14432 let tmp = tempfile::tempdir().unwrap();
14433 let repo = tmp.path().join("repo");
14434 std::fs::create_dir_all(&repo).unwrap();
14435 init_repo(&repo);
14436 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14437 git::git(&repo, &["checkout", "-b", "magi/history/A"])
14438 .await
14439 .unwrap();
14440 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14441 git::git(&repo, &["add", "."]).await.unwrap();
14442 git::git(&repo, &["commit", "-m", "historical implementation"])
14443 .await
14444 .unwrap();
14445 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14446 git::git(&repo, &["checkout", "main"]).await.unwrap();
14447 for instruction in [
14448 format!("Example: magi/history/A at {source}"),
14449 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14450 format!("Never import magi/history/A or {source}"),
14451 format!("Conversation answer: I mentioned {source} only as background"),
14452 ] {
14453 let mut runner = runner_at(RunStatus::Prep);
14454 runner.state.repo = repo.clone();
14455 runner.state.base_commit = base.clone();
14456 runner.state.instruction = instruction.clone();
14457 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14458 runner.state.config.blind.commit_msg_hook = false;
14459 runner.state.seeds = vec![refs::Seed {
14460 token: "magi/history/A".into(),
14461 sha: source.clone(),
14462 kind: refs::SeedKind::Unmerged,
14463 branch: true,
14464 detail: String::new(),
14465 }];
14466 runner.roles.implementers = vec![spec("one")];
14467 assert_eq!(seeded_instruction(&runner.state), instruction);
14468 let mut legacy = serde_json::to_value(&runner.state).unwrap();
14470 legacy.as_object_mut().unwrap().remove("sources");
14471 runner.state = serde_json::from_value(legacy).unwrap();
14472 assert_eq!(runner.state.sources, refs::Sources::default());
14473 runner.state.save().unwrap();
14474 runner.state = RunState::load(&runner.state.id).unwrap();
14475 runner.prep().await.unwrap();
14476 for candidate in &runner.state.candidates {
14477 assert_eq!(
14478 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14479 base
14480 );
14481 assert!(!candidate.worktree.join("unwanted").exists());
14482 }
14483 }
14484 }
14485
14486 #[tokio::test]
14487 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14488 ask_test_home();
14489 let tmp = tempfile::tempdir().unwrap();
14490 let repo = tmp.path().join("repo");
14491 std::fs::create_dir_all(&repo).unwrap();
14492 init_repo(&repo);
14493 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14494 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14495 std::fs::write(repo.join("referenced"), "required commit").unwrap();
14496 git::git(&repo, &["add", "."]).await.unwrap();
14497 git::git(&repo, &["commit", "-m", "required seed"])
14498 .await
14499 .unwrap();
14500 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14501 git::git(&repo, &["checkout", "main"]).await.unwrap();
14502 let mut runner = runner_at(RunStatus::Prep);
14503 runner.state.repo = repo.clone();
14504 runner.state.base_commit = base.clone();
14505 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14506 runner.state.config.graph.advise = false;
14507 runner.state.config.blind.commit_msg_hook = false;
14508 runner.state.sources =
14509 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14510 .await
14511 .unwrap();
14512 runner.roles.implementers = vec![spec("one"), spec("two")];
14513 runner.roles.judges = vec![spec("judge")];
14514 let label = blind::assign_labels(2, runner.state.seed)[0];
14515 let branch = runner.state.branch_for(label);
14516 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14517 git::worktree_add_branch(&repo, &worktree, &branch, &base)
14518 .await
14519 .unwrap();
14520 runner.state.candidates.push(Candidate {
14521 index: 0,
14522 label,
14523 agent: "one".into(),
14524 branch,
14525 worktree,
14526 summary: String::new(),
14527 stat: String::new(),
14528 files: 0,
14529 commits: 0,
14530 empty: false,
14531 failed: None,
14532 verified_noop: None,
14533 duration_ms: 0,
14534 folded: false,
14535 });
14536 runner.state.save().unwrap();
14537 runner.state = RunState::load(&runner.state.id).unwrap();
14539 assert_eq!(runner.state.prep_complete, Some(false));
14540 runner.prep().await.unwrap();
14541 assert_eq!(runner.state.prep_complete, Some(true));
14542 assert_eq!(runner.state.candidates.len(), 2);
14543 for candidate in &runner.state.candidates {
14544 assert_eq!(
14545 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14546 git::tree_of(&repo, &seed).await.unwrap()
14547 );
14548 assert_eq!(
14549 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14550 "required commit"
14551 );
14552 }
14553 assert!(runner.state.worktree_root().join("judge-1").exists());
14554 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14555 runner.prep().await.unwrap();
14556 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14557
14558 runner.state.prep_complete = Some(false);
14559 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14560 run: Some(format!(
14561 "test -f '{}'",
14562 tmp.path().join("dependency-ready").display()
14563 )),
14564 ..Default::default()
14565 }];
14566 assert!(runner.prep().await.is_err());
14567 assert!(runner.state.prep_failure.is_some());
14568 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14569 runner.state = RunState::load(&runner.state.id).unwrap();
14570 assert!(
14571 runner.prep().await.is_err(),
14572 "correcting the cause alone must not auto-retry"
14573 );
14574 let failed_at = runner.state.events.last().unwrap().at;
14575 assert!(
14576 runner.retry_preparation(failed_at).is_err(),
14577 "old approvals cannot unlock the hold"
14578 );
14579 let mut task = crate::queue::Task::new(
14580 "retry".into(),
14581 "retry setup".into(),
14582 repo.clone(),
14583 crate::queue::Source::Human,
14584 );
14585 task.start(runner.state.id.clone());
14586 task.hold_machine(Some("setup failed".into()));
14587 task.release_operator("cli", None);
14588 runner
14589 .retry_preparation(task.resumption.as_ref().unwrap().at)
14590 .unwrap();
14591 runner.prep().await.unwrap();
14592 assert!(runner.state.prep_failure.is_none());
14593 assert_eq!(runner.state.prep_complete, Some(true));
14594 assert_eq!(runner.state.candidates.len(), 2);
14595 assert!(runner.state.worktree_root().join("judge-1").exists());
14596 for candidate in &runner.state.candidates {
14597 assert_eq!(
14598 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14599 git::tree_of(&repo, &seed).await.unwrap()
14600 );
14601 }
14602 }
14603
14604 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14605 let mut heads = Vec::new();
14606 for candidate in candidates {
14607 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14608 }
14609 heads
14610 }
14611
14612 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14613 ask_test_home();
14614 let tmp = tempfile::tempdir().unwrap();
14615 let repo = tmp.path().join("repo");
14616 std::fs::create_dir(&repo).unwrap();
14617 init_repo(&repo);
14618 let mut runner = runner_at(RunStatus::Reviewing);
14619 runner.state.repo = repo.clone();
14620 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14621 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14622 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14623 let branch = format!("candidate-{label}");
14624 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14625 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14626 .await
14627 .unwrap();
14628 if label == 'B' {
14629 std::fs::write(worktree.join("change"), "winner").unwrap();
14630 git::rescue_commit(&worktree, "candidate change")
14631 .await
14632 .unwrap();
14633 }
14634 runner.state.candidates.push(Candidate {
14635 index,
14636 label,
14637 agent: "alpha".into(),
14638 branch,
14639 worktree,
14640 summary: "completed reply".into(),
14641 stat: String::new(),
14642 files: usize::from(label == 'B'),
14643 commits: usize::from(label == 'B'),
14644 empty: label == 'A',
14645 failed: None,
14646 verified_noop: None,
14647 duration_ms: 1,
14648 folded: false,
14649 });
14650 }
14651 runner.state.judge_skipped = true;
14652 runner.state.tally = Some(Tally {
14653 first_choice: BTreeMap::new(),
14654 borda: BTreeMap::new(),
14655 winner: 'B',
14656 rankings: 0,
14657 unanimous_initial: false,
14658 deliberated: false,
14659 changed_votes: 0,
14660 unanimous_final: false,
14661 tie_break: None,
14662 judges: 0,
14663 present: 0,
14664 quorum: 0,
14665 met_quorum: true,
14666 uncontested: Some("only B produced a change".into()),
14667 });
14668 runner.state.save().unwrap();
14669 (tmp, runner)
14670 }
14671
14672 #[tokio::test]
14673 #[cfg(unix)]
14674 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14675 for (solo, failed_checkpoint) in
14676 [(true, false), (false, false), (true, true), (false, true)]
14677 {
14678 let (_tmp, mut runner) = preservation_resume_fixture().await;
14679 if solo {
14680 runner.state.candidates.retain(|c| c.label == 'A');
14681 }
14682 runner.state.candidates[0].empty = false;
14683 runner.state.candidates[0].summary.clear();
14684 runner.state.tally = None;
14685 runner.state.judge_skipped = false;
14686 runner.state.status = RunStatus::Implementing;
14687 runner.state.config.disk.min_free_bytes = u64::MAX;
14688 let mut implementer = spec("alpha");
14689 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14690 runner.roles.implementers = vec![implementer];
14691 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14692 if failed_checkpoint {
14693 std::fs::create_dir_all(&patch).unwrap();
14694 }
14695 runner.state.save().unwrap();
14696 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14697 runner.state = RunState::load(&runner.state.id).unwrap();
14698 assert_eq!(runner.state.status, RunStatus::Blocked);
14699 assert!(runner.state.interrupted_implementations.contains(&'A'));
14700 assert_eq!(runner.state.candidates[0].commits, 0);
14701 assert!(!runner.state.candidates[0].empty);
14702 assert!(runner.state.candidates[0].failed.is_none());
14703 if failed_checkpoint {
14704 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14705 std::fs::remove_dir(patch).unwrap();
14706 }
14707 runner.state.resource_block = None;
14708 runner.state.config.disk.min_free_bytes = 0;
14709 runner.roles.implementers[0].command = vec![
14710 "sh".into(),
14711 "-c".into(),
14712 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14713 ];
14714 runner.implement().await.unwrap();
14715 assert!(runner.state.interrupted_implementations.is_empty());
14716 assert!(runner.state.candidate_preservation.is_empty());
14717 assert_eq!(runner.state.candidates[0].commits, 1);
14718 assert!(!runner.state.candidates[0].empty);
14719 assert!(
14720 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14721 .unwrap()
14722 .contains("retried empty interrupted seat")
14723 );
14724 if !solo {
14725 assert_eq!(runner.state.candidates[1].commits, 1);
14726 }
14727 }
14728 }
14729
14730 #[tokio::test]
14731 #[cfg(unix)]
14732 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14733 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14734 let (_tmp, mut runner) = preservation_resume_fixture().await;
14735 runner.state.candidates.retain(|c| c.label == 'A');
14736 runner.state.candidates[0].empty = false;
14737 runner.state.candidates[0].summary.clear();
14738 runner.state.tally = None;
14739 runner.state.judge_skipped = false;
14740 runner.state.status = RunStatus::Implementing;
14741 runner.state.config.disk.min_free_bytes = u64::MAX;
14744 let wt = runner.state.candidates[0].worktree.clone();
14745 std::fs::write(
14746 wt.join("interrupted-source"),
14747 "retain implementation work\n",
14748 )
14749 .unwrap();
14750 if committed {
14751 git::rescue_commit(&wt, "implementation checkpoint")
14752 .await
14753 .unwrap();
14754 }
14755 let lock = git::git(
14756 &wt,
14757 &[
14758 "rev-parse",
14759 "--path-format=absolute",
14760 "--git-path",
14761 "index.lock",
14762 ],
14763 )
14764 .await
14765 .unwrap();
14766 if block_rescue {
14767 std::fs::write(&lock, "fixture-owned lock").unwrap();
14768 }
14769 let mut implementer = spec("alpha");
14770 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14771 runner.roles.implementers = vec![implementer];
14772 runner.state.save().unwrap();
14773 let result = runner.implement().await;
14774 assert_eq!(result.is_err(), block_rescue);
14775 assert_eq!(
14776 runner.state.resource_block.as_ref().unwrap().kind,
14777 "disk pressure"
14778 );
14779 assert!(
14780 runner
14781 .state
14782 .events
14783 .iter()
14784 .any(|e| e.message.contains("agent execution interrupted"))
14785 );
14786 assert!(
14787 runner
14788 .state
14789 .dir()
14790 .join("artifacts/candidate-A-recovery/staged.patch")
14791 .exists()
14792 );
14793 if block_rescue {
14794 let saved = RunState::load(&runner.state.id).unwrap();
14795 assert!(saved.candidate_preservation.contains_key(&'A'));
14796 assert!(!saved.candidates[0].empty);
14797 assert_eq!(
14798 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14799 "retain implementation work\n"
14800 );
14801 std::fs::remove_file(lock).unwrap();
14802 runner.state = saved;
14803 } else {
14804 assert_eq!(runner.state.status, RunStatus::Blocked);
14805 assert_eq!(runner.state.candidates[0].commits, 1);
14806 }
14807 runner.roles.implementers.clear();
14810 runner.state.config.disk.min_free_bytes = 0;
14811 runner.state.resource_block = None;
14812 runner.implement().await.unwrap();
14813 assert!(runner.state.candidate_preservation.is_empty());
14814 assert_eq!(runner.state.candidates[0].commits, 1);
14815 assert!(!runner.state.candidates[0].empty);
14816 assert!(git::is_clean(&wt).await.unwrap());
14817 assert!(
14818 crate::run::read_artifact(&runner.state, "cand-A.patch")
14819 .unwrap()
14820 .contains("retain implementation work")
14821 );
14822 }
14823 }
14824
14825 #[tokio::test]
14826 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14827 let (_tmp, mut runner) = preservation_resume_fixture().await;
14828 runner.implement().await.unwrap();
14830 assert!(runner.state.candidate_preservation.is_empty());
14831 assert!(
14832 !runner
14833 .state
14834 .dir()
14835 .join("artifacts/candidate-A-recovery")
14836 .exists()
14837 );
14838 runner.fold_losers().await.unwrap();
14839 assert!(!runner.state.candidates[0].worktree.exists());
14840 for schema in [14, crate::run::SCHEMA] {
14841 runner.state.schema = schema;
14842 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14843 runner.state.save().unwrap();
14844 runner.state = RunState::load(&runner.state.id).unwrap();
14845 runner.implement().await.unwrap();
14846 assert!(runner.state.candidate_preservation.is_empty());
14847 assert!(runner.state.candidates[0].folded);
14848 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14849 }
14850 }
14851
14852 #[tokio::test]
14853 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14854 let (_tmp, mut runner) = preservation_resume_fixture().await;
14855 runner.state.candidates.retain(|c| c.label == 'A');
14856 runner.state.tally = None;
14857 runner.state.judge_skipped = false;
14858 let candidate = &mut runner.state.candidates[0];
14859 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14860 candidate.failed = Some("deployment validation unavailable".into());
14861 std::fs::write(
14862 candidate.worktree.join("repair"),
14863 "future launcher policy\n",
14864 )
14865 .unwrap();
14866 runner
14867 .state
14868 .candidate_preservation
14869 .insert('A', "awaiting persistence".into());
14870 runner.persist_candidates().await.unwrap();
14871 let candidate = &runner.state.candidates[0];
14872 assert!(!candidate.empty);
14873 assert_eq!(candidate.commits, 1);
14874 assert_eq!(candidate.files, 1);
14875 assert!(candidate.failed.is_none());
14876 assert!(candidate.summary.contains("EPERM"));
14877 assert!(
14878 crate::run::read_artifact(&runner.state, "cand-A.patch")
14879 .unwrap()
14880 .contains("future launcher policy")
14881 );
14882 assert!(runner.state.candidate_preservation.is_empty());
14883 }
14884
14885 #[tokio::test]
14886 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14887 let (_tmp, mut runner) = preservation_resume_fixture().await;
14888 let wt = runner.state.candidates[0].worktree.clone();
14889 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14890 runner.state.schema = 14;
14891 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14892 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14893 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14894 for _ in 0..2 {
14895 assert!(runner.implement().await.is_err());
14896 assert_eq!(runner.state.status, RunStatus::Blocked);
14897 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14898 assert!(
14899 runner.state.candidates[0].empty,
14900 "old verdict must remain unusable"
14901 );
14902 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14903 assert!(runner.state.judge_skipped);
14904 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14905 assert!(
14906 crate::run::read_artifact(&runner.state, "cand-A.patch")
14907 .unwrap()
14908 .contains("owned edits")
14909 );
14910 assert!(wt.exists());
14911 assert_eq!(
14912 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14913 .await
14914 .unwrap(),
14915 1
14916 );
14917 runner.state = RunState::load(&runner.state.id).unwrap();
14918 }
14919 }
14920
14921 #[tokio::test]
14922 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14923 let (_tmp, mut runner) = preservation_resume_fixture().await;
14924 let wt = runner.state.candidates[0].worktree.clone();
14925 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14926 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14927 let index_lock = PathBuf::from(
14928 git::git(
14929 &wt,
14930 &[
14931 "rev-parse",
14932 "--path-format=absolute",
14933 "--git-path",
14934 "index.lock",
14935 ],
14936 )
14937 .await
14938 .unwrap(),
14939 );
14940 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14941 runner.state.schema = 14;
14942 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14943 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14944 assert!(runner.implement().await.is_err());
14945 assert!(runner.after_implement().is_err());
14946 assert!(runner.fold_losers().await.is_err());
14947 assert_eq!(
14948 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14949 "failed seat edits"
14950 );
14951 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14952 runner.state = RunState::load(&runner.state.id).unwrap();
14953 std::fs::remove_file(index_lock).unwrap();
14954 assert!(runner.implement().await.is_err());
14956 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14957 assert!(runner.state.candidates[0].empty);
14958 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14959 assert!(
14960 crate::run::read_artifact(&runner.state, "cand-A.patch")
14961 .unwrap()
14962 .contains("failed seat edits")
14963 );
14964 assert!(wt.exists());
14965 }
14966
14967 #[tokio::test]
14968 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14969 let (_tmp, mut runner) = preservation_resume_fixture().await;
14970 let repo = runner.state.repo.clone();
14971 let common = git::git(
14972 &repo,
14973 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14974 )
14975 .await
14976 .unwrap();
14977 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14978 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14979 assert!(runner.fold_losers().await.is_err());
14980 assert!(!runner.state.candidates[0].worktree.exists());
14981 assert!(!runner.state.candidates[0].folded);
14982 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14983 runner.state = RunState::load(&runner.state.id).unwrap();
14984 std::fs::remove_file(branch_lock).unwrap();
14985 let judge = runner.state.worktree_root().join("judge-1");
14986 git::worktree_add_detached(&repo, &judge, "main")
14987 .await
14988 .unwrap();
14989 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14990 runner.roles.judges = vec![spec("judge")];
14991 assert!(runner.fold_losers().await.is_err());
14992 runner.state = RunState::load(&runner.state.id).unwrap();
14993 assert!(
14994 runner.state.candidates[0].folded,
14995 "candidate progress saved before judge failure"
14996 );
14997 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14998 assert!(judge.join("uncommitted").exists());
14999 std::fs::remove_file(judge.join("uncommitted")).unwrap();
15000 runner.fold_losers().await.unwrap();
15001 assert!(!judge.exists());
15002 runner.fold_losers().await.unwrap();
15003 }
15004
15005 #[tokio::test]
15006 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15007 let (_tmp, mut runner) = preservation_resume_fixture().await;
15008 runner.state.candidates.retain(|c| c.label == 'B');
15009 runner.state.tally = None;
15010 runner.state.judge_skipped = false;
15011 runner.state.prep_complete = Some(true);
15012 runner.state.config.graph.advise = false;
15013 runner.state.config.graph.selected_implementer = Some("alpha".into());
15014 let wt = runner.state.candidates[0].worktree.clone();
15015 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15016 runner
15017 .state
15018 .candidate_preservation
15019 .insert('B', "awaiting persistence".into());
15020 let index_lock = git::git(
15021 &wt,
15022 &[
15023 "rev-parse",
15024 "--path-format=absolute",
15025 "--git-path",
15026 "index.lock",
15027 ],
15028 )
15029 .await
15030 .unwrap();
15031 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15032 runner.state.save().unwrap();
15033 assert!(runner.execute_graph().await.is_err());
15034 assert_eq!(runner.state.status, RunStatus::Blocked);
15035 assert!(runner.state.candidate_preservation.contains_key(&'B'));
15036 std::fs::remove_file(index_lock).unwrap();
15037 for _ in 0..2 {
15040 runner.state = RunState::load(&runner.state.id).unwrap();
15041 runner.pause.park();
15042 runner.execute_graph().await.unwrap();
15043 assert_eq!(runner.state.status, RunStatus::Implementing);
15044 assert!(runner.state.candidate_preservation.is_empty());
15045 assert_eq!(runner.state.candidates[0].commits, 2);
15046 assert!(runner.state.tally.is_none());
15047 }
15048 runner.implement().await.unwrap();
15049 assert_eq!(runner.state.status, RunStatus::Judging);
15050 runner.state.status = RunStatus::Blocked;
15052 runner.state.candidates[0].failed = Some("implementation failed".into());
15053 runner.state.save().unwrap();
15054 let error = runner.execute_graph().await.unwrap_err();
15055 assert!(
15056 error
15057 .to_string()
15058 .contains("selected implementation is held")
15059 );
15060 }
15061
15062 #[tokio::test]
15063 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15064 let (_tmp, mut runner) = preservation_resume_fixture().await;
15065 let repo = runner.state.repo.clone();
15066 let candidate = runner.state.candidates[0].clone();
15067 let registered = git::worktree_holding(&repo, &candidate.branch)
15070 .await
15071 .unwrap()
15072 .unwrap();
15073 assert_eq!(
15074 registered.canonicalize().unwrap(),
15075 candidate.worktree.canonicalize().unwrap()
15076 );
15077 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15078 assert_eq!(
15079 git::worktree_holding(&repo, &candidate.branch)
15080 .await
15081 .unwrap(),
15082 Some(registered)
15083 );
15084 assert!(
15085 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15086 "the stale registration initially keeps the branch checked out"
15087 );
15088 runner.fold_losers().await.unwrap();
15089 assert!(runner.state.candidates[0].folded);
15090 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15091 assert_eq!(
15092 git::worktree_holding(&repo, &candidate.branch)
15093 .await
15094 .unwrap(),
15095 None
15096 );
15097 runner.state = RunState::load(&runner.state.id).unwrap();
15098 runner.fold_losers().await.unwrap();
15099 }
15100
15101 #[tokio::test]
15102 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15103 let (_tmp, mut runner) = preservation_resume_fixture().await;
15104 let repo = runner.state.repo.clone();
15105 let candidate = runner.state.candidates[0].clone();
15106 let winner = runner.state.candidates[1].clone();
15107 let registered = git::worktree_holding(&repo, &candidate.branch)
15108 .await
15109 .unwrap()
15110 .unwrap();
15111 assert_eq!(
15112 registered.canonicalize().unwrap(),
15113 candidate.worktree.canonicalize().unwrap()
15114 );
15115 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15116 assert_eq!(
15117 git::worktree_holding(&repo, &candidate.branch)
15118 .await
15119 .unwrap(),
15120 Some(registered)
15121 );
15122 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15123
15124 for _ in 0..2 {
15125 fold_run(&mut runner.state, false, &crate::run::home())
15126 .await
15127 .unwrap();
15128 runner.state = RunState::load(&runner.state.id).unwrap();
15129 assert!(runner.state.candidates[0].folded);
15130 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15131 assert_eq!(
15132 git::worktree_holding(&repo, &candidate.branch)
15133 .await
15134 .unwrap(),
15135 None
15136 );
15137 assert!(!runner.state.candidates[1].folded);
15138 assert!(winner.worktree.exists());
15139 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15140 }
15141 }
15142
15143 #[tokio::test]
15144 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15145 ask_test_home();
15146 let tmp = tempfile::tempdir().unwrap();
15147 let repo = tmp.path().join("repo");
15148 std::fs::create_dir(&repo).unwrap();
15149 init_repo(&repo);
15150 let mut runner = runner_at(RunStatus::Implementing);
15151 runner.state.repo = repo.clone();
15152 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15153 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15154 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15155 let branch = format!("candidate-{label}");
15156 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15157 git::worktree_add_branch(&repo, &wt, &branch, "main")
15158 .await
15159 .unwrap();
15160 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15161 git::git(&wt, &["add", "README.md"]).await.unwrap();
15162 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15163 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15164 runner.state.candidates.push(Candidate {
15165 index,
15166 label,
15167 agent: "alpha".to_owned(),
15168 branch,
15169 worktree: wt,
15170 summary: "completed implementation".to_owned(),
15171 stat: String::new(),
15172 files: 0,
15173 commits: 0,
15174 empty: false,
15175 failed: None,
15176 verified_noop: None,
15177 duration_ms: 42,
15178 folded: false,
15179 });
15180 runner
15181 .state
15182 .candidate_preservation
15183 .insert(label, "awaiting persistence".to_owned());
15184 }
15185 let wt = runner.state.candidates[0].worktree.clone();
15186 let index_lock = git::git(
15187 &wt,
15188 &[
15189 "rev-parse",
15190 "--path-format=absolute",
15191 "--git-path",
15192 "index.lock",
15193 ],
15194 )
15195 .await
15196 .unwrap();
15197 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15198 runner.state.save().unwrap();
15199 assert!(runner.persist_candidates().await.is_err());
15200 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15201 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15202 assert_eq!(
15203 runner.state.candidates[1].commits, 1,
15204 "other results are still collected"
15205 );
15206 assert!(
15207 !runner.state.candidates[0].empty,
15208 "failed rescue is not no change"
15209 );
15210 assert_eq!(
15211 runner.state.candidates[0].summary,
15212 "completed implementation"
15213 );
15214 assert!(runner.after_implement().is_err());
15215 let home = tmp.path().join("fold-home");
15216 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15217 assert!(wt.join("new-file").exists());
15218 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15219 assert!(recovery.join("staged.patch").exists());
15220 assert!(recovery.join("unstaged.patch").exists());
15221 assert_eq!(
15222 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15223 "untracked edits\n"
15224 );
15225 runner.state.begin_pass();
15226 runner.state.end_pass_parked(ParkCause::Upgrade {
15227 from: "1.0".into(),
15228 to: Some("1.1".into()),
15229 });
15230 runner.state.save().unwrap();
15231 runner.state = RunState::load(&runner.state.id).unwrap();
15232 assert!(matches!(
15233 runner.state.passes.last().unwrap().end,
15234 Some(crate::run::PassEnd::Parked {
15235 cause: ParkCause::Upgrade { .. }
15236 })
15237 ));
15238 std::fs::remove_file(index_lock).unwrap();
15239 runner.implement().await.unwrap();
15241 assert!(runner.state.candidate_preservation.is_empty());
15242 assert!(
15243 runner
15244 .state
15245 .candidates
15246 .iter()
15247 .all(|c| c.commits == 1 && !c.empty)
15248 );
15249 assert!(git::is_clean(&wt).await.unwrap());
15250 assert!(
15251 crate::run::read_artifact(&runner.state, "cand-A.patch")
15252 .unwrap()
15253 .contains("later tracked edits")
15254 );
15255 }
15256
15257 #[tokio::test]
15258 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15259 ask_test_home();
15260 let tmp = tempfile::tempdir().unwrap();
15261 let repo = tmp.path().join("repo");
15262 std::fs::create_dir(&repo).unwrap();
15263 init_repo(&repo);
15264 let wt = tmp.path().join("candidate");
15265 git::worktree_add_branch(&repo, &wt, "candidate", "main")
15266 .await
15267 .unwrap();
15268 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15269 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15270 assert!(wt.join("uncommitted").exists());
15271 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15272 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15273 let mut runner = runner_at(RunStatus::Implementing);
15274 runner.state.repo = repo.clone();
15275 runner.state.base_commit = "missing-base".to_owned();
15276 runner.state.candidates.push(Candidate {
15277 index: 0,
15278 label: 'A',
15279 agent: "alpha".into(),
15280 branch: "candidate".into(),
15281 worktree: wt.clone(),
15282 summary: "completed".into(),
15283 stat: String::new(),
15284 files: 0,
15285 commits: 0,
15286 empty: false,
15287 failed: None,
15288 verified_noop: None,
15289 duration_ms: 1,
15290 folded: false,
15291 });
15292 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15293 assert!(
15294 fold_run(&mut runner.state, true, &tmp.path().join("home"))
15295 .await
15296 .is_err()
15297 );
15298 assert!(!runner.state.candidates[0].folded);
15299 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15300 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15301 runner
15302 .state
15303 .candidate_preservation
15304 .insert('A', "pending".into());
15305 assert!(runner.persist_candidates().await.is_err());
15306 assert!(!runner.state.candidates[0].empty);
15307 assert!(!runner.state.candidates[0].folded);
15308 assert!(wt.exists());
15309 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15310 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15311 std::fs::create_dir_all(&patch_path).unwrap();
15312 assert!(
15313 runner.persist_candidates().await.is_err(),
15314 "artifact failure stays pending"
15315 );
15316 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15317 assert!(!runner.state.candidates[0].empty);
15318 std::fs::remove_dir(patch_path).unwrap();
15319 runner.persist_candidates().await.unwrap();
15320 assert!(
15321 runner.state.candidates[0].empty,
15322 "only successful inspection confirms no change"
15323 );
15324 }
15325
15326 fn init_repo(dir: &Path) {
15327 let run = |args: &[&str]| {
15328 let out = std::process::Command::new("git")
15329 .args(args)
15330 .current_dir(dir)
15331 .quiet()
15332 .output()
15333 .expect("spawn git");
15334 assert!(
15335 out.status.success(),
15336 "git {args:?} failed: {}",
15337 String::from_utf8_lossy(&out.stderr)
15338 );
15339 };
15340 run(&["init", "-b", "main"]);
15341 run(&["config", "user.name", "magi test"]);
15342 run(&["config", "user.email", "magi@example.com"]);
15343 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15344 run(&["add", "-A"]);
15345 run(&["commit", "-m", "init"]);
15346 }
15347
15348 fn ask_test_home() {
15356 crate::run::pin_test_home();
15357 }
15358
15359 #[cfg(unix)]
15360 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15361 ask_test_home();
15362 let tmp = tempfile::tempdir().unwrap();
15363 let repo = tmp.path().join("repo");
15364 std::fs::create_dir(&repo).unwrap();
15365 init_repo(&repo);
15366 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15367 let origin = tmp.path().join("origin.git");
15368 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15369 .await
15370 .unwrap();
15371 git::git(
15372 &repo,
15373 &["remote", "add", "origin", origin.to_str().unwrap()],
15374 )
15375 .await
15376 .unwrap();
15377 git::git(&repo, &["checkout", "-b", "fix/retained"])
15378 .await
15379 .unwrap();
15380 std::fs::write(repo.join("retained"), "retained change").unwrap();
15381 git::git(&repo, &["add", "."]).await.unwrap();
15382 git::git(&repo, &["commit", "-m", "retained implementation"])
15383 .await
15384 .unwrap();
15385 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15386 git::git(&repo, &["checkout", "main"]).await.unwrap();
15387 std::fs::write(repo.join("base-change"), "new base").unwrap();
15388 git::git(&repo, &["add", "."]).await.unwrap();
15389 git::git(&repo, &["commit", "-m", "base moved"])
15390 .await
15391 .unwrap();
15392 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15393 git::git(&repo, &["checkout", "fix/retained"])
15394 .await
15395 .unwrap();
15396 let mut runner = runner_at(RunStatus::Voting);
15397 runner.state.repo = repo.clone();
15398 runner.state.base_commit = base;
15399 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15400 runner.state.config.disk.min_free_bytes = 0;
15401 runner.state.config.graph.review_rounds = 1;
15402 runner.state.config.verify.e2e = vec!["true".into()];
15403 runner.state.config.verify.gate = vec!["true".into()];
15404 runner.state.candidates = vec![Candidate {
15405 index: 0,
15406 label: 'A',
15407 agent: "fixture".into(),
15408 branch: "fix/retained".into(),
15409 worktree: repo.clone(),
15410 summary: String::new(),
15411 stat: String::new(),
15412 files: 1,
15413 commits: 1,
15414 empty: false,
15415 failed: None,
15416 verified_noop: None,
15417 duration_ms: 0,
15418 folded: false,
15419 }];
15420 runner.state.tally = Some(Tally {
15421 first_choice: BTreeMap::new(),
15422 borda: BTreeMap::new(),
15423 winner: 'A',
15424 rankings: 0,
15425 unanimous_initial: true,
15426 deliberated: false,
15427 changed_votes: 0,
15428 unanimous_final: true,
15429 tie_break: None,
15430 judges: 0,
15431 present: 0,
15432 quorum: 0,
15433 met_quorum: true,
15434 uncontested: Some("review-only".into()),
15435 });
15436 let mut task = crate::queue::Task::new(
15437 "fixture".into(),
15438 "fixture".into(),
15439 repo,
15440 crate::queue::Source::Human,
15441 );
15442 task.link_run(&runner.state.id);
15443 crate::queue::Queue::open().put(&mut task).unwrap();
15444 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15445 runner.state.base_sync = Some(BaseSync {
15446 tip: String::new(),
15447 behind: 1,
15448 attempts: BASE_SYNC_ROUNDS,
15449 conflict: None,
15450 already_in: None,
15451 });
15452 runner.sync_to_base().await.unwrap();
15453 assert_eq!(runner.state.status, RunStatus::Blocked);
15454 (tmp, runner, head, task.id)
15455 }
15456
15457 #[cfg(unix)]
15458 #[tokio::test]
15459 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15460 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15461 if std::env::var_os(CHILD).is_none() {
15462 let home = tempfile::tempdir().unwrap();
15463 let output = std::process::Command::new(std::env::current_exe().unwrap())
15464 .args([
15465 "--exact",
15466 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15467 "--nocapture",
15468 ])
15469 .env(CHILD, "1")
15470 .env("MAGI_HOME", home.path())
15471 .env("MAGI_WORKER_HOME", home.path())
15472 .env("MAGI_WORKER_ID", "abcdef")
15473 .output()
15474 .unwrap();
15475 assert!(
15476 output.status.success()
15477 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15478 "{}\n{}",
15479 String::from_utf8_lossy(&output.stdout),
15480 String::from_utf8_lossy(&output.stderr)
15481 );
15482 return;
15483 }
15484 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15485 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15486 let home = crate::run::home();
15487 let queue = crate::queue::Queue::open();
15488 let mut owner = queue.get(&task).unwrap();
15489 owner.release_operator("test-release", None);
15490 owner.start(runner.state.id.clone());
15491 queue.put(&mut owner).unwrap();
15492 runner.state.driver_exited = true;
15493 runner.state.task_instruction = Some(owner.instruction.clone());
15494 runner.state.save().unwrap();
15495 let _claim = queue.claim(&task).unwrap();
15496 let reserve = |id: &str| {
15497 let directory = home.join("workers/jobs").join(id);
15498 std::fs::create_dir_all(&directory).unwrap();
15499 let job = crate::worker::Job {
15500 id: id.into(),
15501 sequence: 1,
15502 kind: "task".into(),
15503 key: format!("task:{task}"),
15504 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15505 generation: "fixture-generation".into(),
15506 version: None,
15507 status: crate::worker::Status::Running,
15508 result: None,
15509 pid: Some(std::process::id()),
15510 created_at: 0,
15511 finished_at: None,
15512 };
15513 std::fs::write(
15514 directory.join("job.json"),
15515 serde_json::to_vec(&job).unwrap(),
15516 )
15517 .unwrap();
15518 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15519 };
15520 let _own = reserve("abcdef");
15521 let other = reserve("fedcba");
15522 assert!(
15523 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15524 .await
15525 .is_err()
15526 );
15527 assert!(
15528 RunState::load(&runner.state.id)
15529 .unwrap()
15530 .base_sync_recovery
15531 .authorizations
15532 .is_empty()
15533 );
15534 drop(other);
15535 assert!(crate::daemon::is_working_on(
15536 &home,
15537 &runner.state.id,
15538 Timestamp::now()
15539 ));
15540 assert!(
15541 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15542 );
15543 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15544 .await
15545 .unwrap();
15546 assert!(
15547 RunState::load(&runner.state.id)
15548 .unwrap()
15549 .base_recovery_pending()
15550 );
15551 }
15552
15553 #[cfg(unix)]
15554 #[tokio::test]
15555 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15556 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15557 let home = crate::run::home();
15558 runner.state.driver_pid = Some(std::process::id());
15559 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15560 runner.state.driver_exited = false;
15561 runner.state.begin_pass();
15562 runner.state.save().unwrap();
15563 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15564 driver.record(&runner.state).unwrap();
15565 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15566 drop(driver);
15568 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15569 assert_eq!(
15570 runner.state.liveness_under(&home, false),
15571 crate::run::Liveness::Dead
15572 );
15573 let queue = crate::queue::Queue::open();
15574 let _claim = queue.claim(&task).unwrap();
15575 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15576 .await
15577 .unwrap();
15578 let disk = RunState::load(&runner.state.id).unwrap();
15579 assert!(!disk.driver_exited);
15580 assert!(disk.base_recovery_pending());
15581 }
15582
15583 #[cfg(unix)]
15584 #[tokio::test]
15585 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15586 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15587 if std::env::var_os(CHILD).is_none() {
15588 use std::os::unix::fs::PermissionsExt;
15589 let tmp = tempfile::tempdir().unwrap();
15590 let gh = tmp.path().join("gh");
15591 std::fs::write(&gh, r#"#!/bin/sh
15592case "$1 $2" in
15593 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15594 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15595 *) exit 2 ;;
15596esac
15597"#).unwrap();
15598 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15599 let mut paths = vec![tmp.path().to_path_buf()];
15600 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15601 let output = std::process::Command::new(std::env::current_exe().unwrap())
15602 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15603 .env(CHILD, "1")
15604 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15605 .env("PATH", std::env::join_paths(paths).unwrap())
15606 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15607 .output().unwrap();
15608 assert!(
15609 output.status.success()
15610 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15611 "{}\n{}",
15612 String::from_utf8_lossy(&output.stdout),
15613 String::from_utf8_lossy(&output.stderr)
15614 );
15615 return;
15616 }
15617 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15618 let repo = runner.state.repo.clone();
15619 git::git(&repo, &["checkout", "main"]).await.unwrap();
15620 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15621 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15622 git::git(&repo, &["checkout", "fix/retained"])
15623 .await
15624 .unwrap();
15625 runner.state.pr = Some(crate::run::PrRecord {
15626 url: "https://example.test/project/pull/1".into(),
15627 number: 1,
15628 state: "open".into(),
15629 checks: "unknown".into(),
15630 round: 0,
15631 rounds: 1,
15632 red_at_merge: Vec::new(),
15633 });
15634 runner.sync_to_base().await.unwrap();
15635 assert_eq!(runner.state.status, RunStatus::Blocked);
15636 assert_eq!(
15637 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15638 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15639 );
15640 assert!(runner.state.base_sync_closure_pending());
15641 assert!(crate::handover::can_resume(&runner.state));
15642 let queue = crate::queue::Queue::open();
15643 assert!(!crate::base_sync_recovery::held_task(
15644 &queue,
15645 &queue.get(&task).unwrap()
15646 ));
15647 assert!(runner.state.authorize_base_sync(&task).is_err());
15648 runner.state.base_sync_recovery.stop = None;
15650 runner.state.status = RunStatus::Voting;
15651 runner.state.gate_ran = true;
15652 runner.state.config.graph.review_rounds = 0;
15653 runner.state.save().unwrap();
15654 assert!(crate::handover::can_resume(&runner.state));
15655 runner.state = RunState::load(&runner.state.id).unwrap();
15656 runner.execute_graph().await.unwrap();
15657 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15658 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15659 assert_eq!(
15660 runner.state.base_sync.as_ref().unwrap().attempts,
15661 BASE_SYNC_ROUNDS
15662 );
15663 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15664 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15665 assert!(runner.state.reviews.is_empty());
15666 assert!(runner.state.gate.is_empty());
15667 }
15668
15669 #[cfg(unix)]
15670 #[tokio::test]
15671 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15672 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15673 runner.state.driver_exited = true;
15674 runner.state.save().unwrap();
15675 let home = crate::run::home();
15676 let queue = crate::queue::Queue::open();
15677 let _claim = queue.claim(&task).unwrap();
15678 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15679 assert!(
15680 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15681 .await
15682 .is_err()
15683 );
15684 drop(lease);
15685 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15686 assert!(
15687 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15688 .await
15689 .is_err()
15690 );
15691 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15692 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15693 .await
15694 .unwrap();
15695 let disk = RunState::load(&runner.state.id).unwrap();
15696 assert!(disk.base_recovery_pending());
15697 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15698 assert!(
15699 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15700 .await
15701 .is_err()
15702 );
15703 assert_eq!(
15704 RunState::load(&runner.state.id)
15705 .unwrap()
15706 .base_sync_recovery
15707 .authorizations
15708 .len(),
15709 1
15710 );
15711 let mut owner = queue.get(&task).unwrap();
15712 owner.runs.push("20260101-000000-ffff".into());
15713 queue.put(&mut owner).unwrap();
15714 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15715 .await
15716 .unwrap_err();
15717 assert!(error.to_string().contains("run task ownership changed"));
15718 }
15719
15720 #[cfg(unix)]
15721 #[tokio::test]
15722 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15723 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15724 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15725 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15726 legacy["status"] = serde_json::json!("voting");
15727 runner.state = serde_json::from_value(legacy).unwrap();
15728 runner.state.save().unwrap();
15729 for _ in 0..3 {
15730 runner.execute_graph().await.unwrap();
15731 assert_eq!(runner.state.status, RunStatus::Blocked);
15732 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15733 assert!(!crate::handover::can_resume(&runner.state));
15734 assert_eq!(
15735 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15736 head
15737 );
15738 assert!(runner.state.reviews.is_empty());
15739 assert!(!runner.state.gate_ran);
15740 }
15741 let notice = crate::notices::run_ended(&runner.state).unwrap();
15742 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15743 runner.state.begin_pass();
15744 runner.state.status = RunStatus::Voting;
15745 runner.deliberate().await.unwrap();
15746 runner.vote().await.unwrap();
15747 runner.finish_after_tally().await.unwrap();
15748 assert_eq!(runner.state.status, RunStatus::Blocked);
15749 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15750 assert_eq!(
15751 crate::notices::run_ended(&runner.state).unwrap().since,
15752 notice.since
15753 );
15754 }
15755
15756 #[cfg(unix)]
15757 #[tokio::test]
15758 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15759 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15760 for detail in [
15761 "fix/retained conflicts with origin/main and did not rebase: conflict",
15762 "rebased fix/retained locally but origin refused the push",
15763 "unknown failure",
15764 ] {
15765 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15766 runner.state.record_base_sync_stop(&head);
15767 assert!(runner.state.authorize_base_sync(&task).is_err());
15768 runner.state.status = RunStatus::Voting;
15769 runner.execute_graph().await.unwrap();
15770 assert_eq!(runner.state.status, RunStatus::Blocked);
15771 assert!(!crate::handover::can_resume(&runner.state));
15772 assert_eq!(
15773 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15774 head
15775 );
15776 }
15777 }
15778
15779 #[cfg(unix)]
15780 #[tokio::test]
15781 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15782 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15783 runner.state.authorize_base_sync(&task).unwrap();
15784 assert!(runner.state.consume_base_recovery(&head));
15785 runner.state.save().unwrap();
15786 runner.state = RunState::load(&runner.state.id).unwrap();
15787 runner.execute_graph().await.unwrap();
15788 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15789 assert_eq!(runner.state.status, RunStatus::Blocked);
15790 assert!(!runner.state.consume_base_recovery(&head));
15791 assert!(!crate::handover::can_resume(&runner.state));
15792 runner.state.authorize_base_sync(&task).unwrap();
15793 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15794 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15795 assert_eq!(
15796 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15797 head
15798 );
15799 }
15800
15801 #[cfg(unix)]
15802 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
15803 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
15804 runner.state.prep_complete = Some(true);
15806 runner.state.base_sync = None;
15807 runner.state.base_sync_recovery = Default::default();
15808 runner.state.config.graph.gate_fix_rounds = 1;
15809 let mut review = review_round(true, 0, 1, 1, false, true);
15810 review.head = head.clone();
15811 review.verified_head = Some(head.clone());
15812 runner.state.reviews = vec![review];
15813 runner.state.gate = vec![CommandOutcome {
15814 command: "old gate".into(),
15815 code: Some(1),
15816 output_tail: "old failure".into(),
15817 duration_ms: 0,
15818 resource_blocked: false,
15819 }];
15820 runner.state.gate_head = Some(head.clone());
15821 runner.state.gate_ran = true;
15822 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
15823 agent: "fixture".into(),
15824 failed: runner.state.gate.clone(),
15825 notes: "spent".into(),
15826 committed: false,
15827 error: None,
15828 }];
15829 runner.state.land_approval = Some(crate::run::LandApproval {
15830 question: "old approval".into(),
15831 head: head.clone(),
15832 });
15833 runner.state.status = RunStatus::Blocked;
15834 (tmp, runner, head)
15835 }
15836
15837 #[cfg(unix)]
15838 #[tokio::test]
15839 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
15840 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
15841 let (tmp, mut runner, old) = resumed_gate_fixture().await;
15842 let phases = tmp.path().join("phases");
15843 runner.state.config.verify.e2e = vec![format!(
15844 "echo e2e >> '{}'; {}",
15845 phases.display(),
15846 if e2e_ok {
15847 "true"
15848 } else {
15849 "echo new-e2e-failure; false"
15850 }
15851 )];
15852 runner.state.config.verify.gate = vec![format!(
15853 "echo gate >> '{}'; {}",
15854 phases.display(),
15855 if gate_ok {
15856 "true"
15857 } else {
15858 "echo new-gate-failure; false"
15859 }
15860 )];
15861 runner.state.save().unwrap();
15862 runner.state = RunState::load(&runner.state.id).unwrap();
15863 runner.sync_to_base().await.unwrap();
15864 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15865 assert_ne!(head, old);
15866 runner.state = RunState::load(&runner.state.id).unwrap();
15868 runner.execute_graph().await.unwrap();
15869 assert_eq!(runner.state.gate_fixes.len(), 1);
15870 assert_eq!(runner.state.reviews.len(), 1);
15871 assert_eq!(runner.state.reviews[0].head, old);
15872 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
15873 assert_eq!(runner.state.verification_history.len(), 1);
15874 let history = &runner.state.verification_history[0];
15875 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
15876 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15877 assert!(history.archived_at.is_some());
15878 assert_eq!(history.fixes_spent, Some(1));
15879 assert_eq!(history.gate[0].output_tail, "old failure");
15880 let archive_events: Vec<_> = runner
15881 .state
15882 .events
15883 .iter()
15884 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
15885 .collect();
15886 assert_eq!(archive_events.len(), 1);
15887 assert!(archive_events[0].message.contains(&short(&old)));
15888 assert!(archive_events[0].message.contains(&short(&head)));
15889 assert!(
15890 archive_events[0]
15891 .message
15892 .contains("1 gate-fix round(s) already spent stay spent")
15893 );
15894 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
15895 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
15896 if e2e_ok {
15897 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15898 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
15899 } else {
15900 assert!(!runner.state.gate_ran);
15901 }
15902 assert_eq!(
15903 runner.state.status,
15904 if e2e_ok && gate_ok {
15905 RunStatus::Ready
15906 } else {
15907 RunStatus::Blocked
15908 }
15909 );
15910 if !e2e_ok || !gate_ok {
15911 let cause = runner.state.verification_blocking_cause().unwrap();
15912 assert!(cause.contains(&head));
15913 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
15914 runner.state.event("land", "rebased successfully");
15916 let notice = crate::notices::run_ended(&runner.state).unwrap();
15917 assert!(!format!("{notice:?}").contains("rebased successfully"));
15918 assert!(runner.state.merge.is_none());
15919 }
15920 let before = std::fs::read_to_string(&phases).unwrap();
15921 runner.state.save().unwrap();
15922 runner.state = RunState::load(&runner.state.id).unwrap();
15923 runner.review_loop().await.unwrap();
15924 runner.gate().await.unwrap();
15925 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
15926 assert_eq!(runner.state.verification_history.len(), 1);
15927 let history = &runner.state.verification_history[0];
15928 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15929 assert!(history.archived_at.is_some());
15930 assert_eq!(history.fixes_spent, Some(1));
15931 }
15932 }
15933
15934 #[cfg(unix)]
15935 #[tokio::test]
15936 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
15937 let (tmp, mut runner, head) = resumed_gate_fixture().await;
15938 let marker = tmp.path().join("gate-ran");
15939 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
15940 runner.review_loop().await.unwrap();
15941 runner.gate().await.unwrap();
15942 assert_eq!(runner.state.status, RunStatus::Blocked);
15943 assert!(!marker.exists());
15944 assert!(runner.state.verification_history.is_empty());
15945 runner.state.gate_head = None;
15946 runner.gate().await.unwrap();
15947 assert!(marker.exists());
15948 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15949 assert_eq!(runner.state.verification_history.len(), 1);
15950 assert_eq!(runner.state.gate_fixes.len(), 1);
15951 }
15952
15953 #[cfg(unix)]
15954 #[tokio::test]
15955 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
15956 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
15957 runner.sync_to_base().await.unwrap();
15958 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15959 runner.state.reconcile_verification_head(&head);
15960 for outcomes in [
15962 Vec::new(),
15963 vec![CommandOutcome {
15964 command: "true".into(),
15965 code: None,
15966 output_tail: "cache unavailable".into(),
15967 duration_ms: 0,
15968 resource_blocked: true,
15969 }],
15970 ] {
15971 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
15972 head: head.clone(),
15973 passed: false,
15974 outcomes,
15975 });
15976 runner.state.gate_ran = false;
15977 runner.state.gate.clear();
15978 runner.state.save().unwrap();
15979 runner.state = RunState::load(&runner.state.id).unwrap();
15980 runner.merge().await.unwrap();
15981 assert!(runner.state.merge.is_none());
15982 runner.review_loop().await.unwrap();
15983 runner.gate().await.unwrap();
15984 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
15985 assert!(runner.state.gate_allows_progress());
15986 assert_eq!(runner.state.gate_fixes.len(), 1);
15987 assert!(runner.state.merge.is_none());
15988 }
15989 git::git(
15991 &runner.state.repo,
15992 &["commit", "--allow-empty", "-m", "external update"],
15993 )
15994 .await
15995 .unwrap();
15996 runner.merge().await.unwrap();
15997 assert!(runner.state.merge.is_none());
15998 assert_eq!(runner.state.status, RunStatus::Blocked);
15999 }
16000
16001 #[cfg(unix)]
16002 #[tokio::test]
16003 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16004 for waiting_on_gate in [false, true] {
16005 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16006 runner.sync_to_base().await.unwrap();
16007 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16008 runner.state.reconcile_verification_head(&head);
16009 if waiting_on_gate {
16010 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16011 head: head.clone(),
16012 passed: true,
16013 outcomes: Vec::new(),
16014 });
16015 }
16016 let cache = tmp.path().join("cache");
16017 runner.state.config.verify.e2e =
16018 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16019 runner.state.config.verify.gate = vec!["true".into()];
16020 runner.state.config.graph.timeout_resource_wait = 1;
16021 let owner =
16022 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16023 let guard =
16024 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16025 crate::cache::AcquireOutcome::Acquired(g) => g,
16026 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16027 };
16028 runner.review_loop().await.unwrap();
16029 runner.gate().await.unwrap();
16030 assert!(!runner.state.gate_ran);
16031 assert_eq!(runner.state.gate_fixes.len(), 1);
16032 assert!(runner.state.resource_block.is_some());
16033 runner.merge().await.unwrap();
16034 assert!(runner.state.merge.is_none());
16035 drop(guard);
16036 runner.state = RunState::load(&runner.state.id).unwrap();
16037 runner.state.resource_block = None;
16039 runner.review_loop().await.unwrap();
16040 runner.gate().await.unwrap();
16041 assert!(runner.state.gate_allows_progress());
16042 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16043 assert_eq!(runner.state.gate_fixes.len(), 1);
16044 }
16045 }
16046
16047 #[cfg(unix)]
16048 #[tokio::test]
16049 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16050 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16051 runner.sync_to_base().await.unwrap();
16052 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16053 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16054 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16056 runner.review_loop().await.unwrap();
16057 runner.gate().await.unwrap();
16058 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16059 assert_ne!(before, after);
16060 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16061 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16062 assert!(!runner.state.gate_allows_progress());
16063 runner.merge().await.unwrap();
16064 assert!(runner.state.merge.is_none());
16065 assert!(
16066 runner
16067 .state
16068 .verification_history
16069 .iter()
16070 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16071 );
16072
16073 runner.state.reviews[0].clean = false;
16075 runner.state.reviews[0].answered = 0;
16076 runner.state.config.verify.e2e = vec!["true".into()];
16077 runner.review_loop().await.unwrap();
16078 runner.gate().await.unwrap();
16079 runner.merge().await.unwrap();
16080 assert_eq!(runner.state.status, RunStatus::Blocked);
16081 assert!(runner.state.merge.is_none());
16082 assert!(
16083 runner
16084 .state
16085 .verification_blocking_cause()
16086 .unwrap()
16087 .contains("review blocked")
16088 );
16089 }
16090
16091 #[cfg(unix)]
16092 #[tokio::test]
16093 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16094 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16095 runner.state.gate_head = None;
16096 runner.state.config.verify.gate =
16097 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16098 runner.review_loop().await.unwrap();
16099 runner.gate().await.unwrap();
16100 assert_ne!(
16101 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16102 old
16103 );
16104 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16105 assert!(!runner.state.gate_allows_progress());
16106 assert_eq!(runner.state.status, RunStatus::Blocked);
16107 runner.merge().await.unwrap();
16108 assert!(runner.state.merge.is_none());
16109 }
16110
16111 #[cfg(unix)]
16112 #[tokio::test]
16113 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16114 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16115 let mut old_review = review_round(true, 0, 1, 1, false, true);
16116 old_review.head = head.clone();
16117 old_review.verified_head = Some(head.clone());
16118 runner.state.reviews = vec![old_review];
16119 runner.state.gate = vec![CommandOutcome {
16120 command: "old gate".into(),
16121 code: None,
16122 output_tail: "timed out after 2400s".into(),
16123 duration_ms: 2_400_000,
16124 resource_blocked: false,
16125 }];
16126 runner.state.gate_ran = true;
16127 runner.state.gate_head = Some(head.clone());
16128 runner.state.land_approval = Some(crate::run::LandApproval {
16129 question: "old-head-approval".into(),
16130 head: head.clone(),
16131 });
16132 runner.state.authorize_base_sync(&task).unwrap();
16133 runner.sync_to_base().await.unwrap();
16134 assert!(!runner.state.base_sync_blocked());
16135 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16136 assert_ne!(head, fresh);
16137 assert!(runner.state.reviews.is_empty());
16138 assert!(!runner.state.gate_ran);
16139 let saved = &runner.state.base_sync_recovery.authorizations[0];
16140 assert_eq!(saved.reviews[0].head, head);
16141 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16142 assert!(saved.gate_ran);
16143 assert!(!saved.gate[0].ok());
16144 runner.review_loop().await.unwrap();
16145 runner.gate().await.unwrap();
16146 assert_eq!(runner.state.reviews[0].head, fresh);
16147 assert_eq!(
16148 runner.state.reviews[0].verified_head.as_deref(),
16149 Some(fresh.as_str())
16150 );
16151 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16152 assert!(runner.state.gate_ran);
16153 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16154 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16155 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16156 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16157 assert_eq!(
16158 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16159 "retained change"
16160 );
16161 }
16162
16163 #[cfg(unix)]
16164 #[tokio::test]
16165 async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16166 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16167 let queue = crate::queue::Queue::open();
16168 let mut owner = queue.get(&task).unwrap();
16169 owner.release_operator("test-resume", None);
16170 queue.put(&mut owner).unwrap();
16171 let generation = owner.resumption.as_ref().unwrap().generation;
16172 runner.state.task_instruction = Some(owner.instruction.clone());
16173 runner.state.driver_exited = true;
16174 runner.state.save().unwrap();
16175 let home = crate::run::home();
16176 let _claim = queue.claim(&task).unwrap();
16177 owner.resumption.as_mut().unwrap().at =
16178 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16179 queue.put(&mut owner).unwrap();
16180 assert!(
16181 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16182 .await
16183 .is_err(),
16184 "an approval from before the stop grants nothing"
16185 );
16186 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16187 queue.put(&mut owner).unwrap();
16188 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16189 .await
16190 .unwrap();
16191 assert!(
16192 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16193 .await
16194 .is_err()
16195 );
16196 runner.state = RunState::load(&runner.state.id).unwrap();
16197 assert_eq!(
16198 runner.state.base_sync_recovery.authorizations[0].generation,
16199 Some(generation)
16200 );
16201 runner.sync_to_base().await.unwrap();
16202 let repo = runner.state.repo.clone();
16203 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16204 git::git(&repo, &["checkout", "main"]).await.unwrap();
16205 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16206 git::git(&repo, &["add", "."]).await.unwrap();
16207 git::git(&repo, &["commit", "-m", "move base again"])
16208 .await
16209 .unwrap();
16210 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16211 git::git(&repo, &["checkout", "fix/retained"])
16212 .await
16213 .unwrap();
16214 runner.finish_after_tally().await.unwrap();
16215 assert_eq!(runner.state.status, RunStatus::Blocked);
16216 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16217 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16218 assert!(!runner.state.base_recovery_pending());
16219 assert!(!crate::handover::can_resume(&runner.state));
16220 assert!(!runner.state.gate_ran);
16221 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16223 queue.put(&mut owner).unwrap();
16224 assert!(
16225 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16226 .await
16227 .is_err()
16228 );
16229 runner.state = RunState::load(&runner.state.id).unwrap();
16230 runner.sync_to_base().await.unwrap();
16231 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16232 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16233 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16234 }
16235
16236 #[cfg(unix)]
16237 #[tokio::test]
16238 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16239 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16240 runner.state.base_sync = None;
16241 runner.state.status = RunStatus::Blocked;
16242 let mut second = runner.state.candidates[0].clone();
16243 second.label = 'B';
16244 runner.state.candidates.push(second);
16245 assert!(runner.state.votes.is_empty());
16246 assert!(runner.state.deliberation.is_empty());
16247 runner.deliberate().await.unwrap();
16248 runner.vote().await.unwrap();
16249 assert_eq!(runner.state.status, RunStatus::Blocked);
16250 assert!(runner.state.votes.is_empty());
16251 assert!(runner.state.deliberation.is_empty());
16252 }
16253
16254 fn runner_at(status: RunStatus) -> Runner {
16257 let mut state = RunState::new(
16258 PathBuf::from("/nonexistent/repo"),
16259 "main".to_owned(),
16260 "deadbeef".to_owned(),
16261 "task".to_owned(),
16262 Config::default(),
16263 );
16264 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16267 state.id = format!(
16268 "20000101-000000-{:04x}",
16269 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16270 );
16271 state.status = status;
16272 Runner {
16273 state,
16274 roles: ResolvedRoles {
16275 implementers: Vec::new(),
16276 judges: Vec::new(),
16277 reviewers: Vec::new(),
16278 fixer: None,
16279 conductor: conductor(),
16280 implementer_roster: Vec::new(),
16281 judge_roster: Vec::new(),
16282 reviewer_roster: Vec::new(),
16283 },
16284 sem: Arc::new(Semaphore::new(1)),
16285 pause: Pause::new(),
16286 interrupt: Pause::new(),
16287 }
16288 }
16289
16290 #[tokio::test]
16291 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16292 crate::run::pin_test_home();
16293 let mut runner = runner_at(RunStatus::Gating);
16294 runner.state.save().unwrap();
16295 let id = runner.state.id.clone();
16296 let passes = runner.state.passes.len();
16297 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16298 let mut claiming = Box::pin(runner.claim_driver());
16299 std::future::poll_fn(|cx| {
16300 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16301 std::task::Poll::Ready(())
16302 })
16303 .await;
16304 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16305 drop(owner);
16306 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16307 .await
16308 .unwrap()
16309 .unwrap();
16310 assert_eq!(execution.runner.state.passes.len(), passes + 1);
16311 assert!(execution.runner.state.resource_block.is_none());
16312 assert_eq!(
16313 execution.runner.state.liveness(false),
16314 crate::run::Liveness::Live
16315 );
16316 drop(execution);
16317 assert!(RunState::load(&id).unwrap().driver_exited);
16318 }
16319
16320 #[tokio::test]
16321 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16322 crate::run::pin_test_home();
16323 let mut runner = runner_at(RunStatus::Gating);
16324 runner.state.save().unwrap();
16325 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16328 assert!(runner.claim_driver().await.is_err());
16329 assert!(runner.state.driver_exited);
16330 assert_eq!(runner.state.driver_pid, None);
16331 let disk = RunState::load(&runner.state.id).unwrap();
16332 assert!(disk.driver_exited);
16333 assert_eq!(disk.driver_pid, None);
16334 assert_eq!(disk.status, RunStatus::Gating);
16335 }
16336
16337 #[tokio::test]
16338 async fn an_early_graph_error_releases_its_driver() {
16339 crate::run::pin_test_home();
16340 let mut runner = runner_at(RunStatus::Blocked);
16341 runner.state.config.graph.selected_implementer = Some("selected".into());
16342 runner.state.save().unwrap();
16343 assert!(runner.execute().await.is_err());
16344 let disk = RunState::load(&runner.state.id).unwrap();
16345 assert!(disk.driver_exited);
16346 assert_eq!(disk.driver_pid, None);
16347 assert_eq!(disk.status, RunStatus::Blocked);
16348 }
16349
16350 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16351 crate::run::MergeRepair {
16352 question: "repair-question".into(),
16353 head: "old-head".into(),
16354 pr: "https://github.com/o/r/pull/42".into(),
16355 reviews: vec![review_round(false, 1, 1, 1, false, true)],
16356 gate: vec![outcome(Some(0), false)],
16357 gate_head: Some("old-head".into()),
16358 gate_fixes: Vec::new(),
16359 pre_gate: Vec::new(),
16360 phase,
16361 failure: None,
16362 }
16363 }
16364
16365 #[tokio::test]
16366 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16367 crate::run::pin_test_home();
16368 let mut runner = runner_at(RunStatus::Landing);
16369 runner.state.merge_repairs.push(merge_repair_request(
16370 crate::run::MergeRepairPhase::Requested,
16371 ));
16372 runner.state.save().unwrap();
16373 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16374 assert!(
16375 runner
16376 .execute()
16377 .await
16378 .unwrap_err()
16379 .to_string()
16380 .contains("already running")
16381 );
16382 assert_eq!(
16383 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16384 crate::run::MergeRepairPhase::Requested
16385 );
16386 drop(claim);
16387 let mut disk = runner.state.clone();
16388 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16389 disk.save().unwrap();
16390 assert!(
16391 runner
16392 .execute()
16393 .await
16394 .unwrap_err()
16395 .to_string()
16396 .contains("decision changed")
16397 );
16398 assert_eq!(
16399 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16400 crate::run::MergeRepairPhase::Complete
16401 );
16402 }
16403
16404 #[test]
16405 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16406 crate::run::pin_test_home();
16407 let mut runner = runner_at(RunStatus::Landing);
16408 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16409 runner.state.reviews = request.reviews.clone();
16410 runner.state.gate = request.gate.clone();
16411 runner.state.gate_ran = true;
16412 runner.state.land_approval = Some(crate::run::LandApproval {
16413 question: "old".into(),
16414 head: "old-head".into(),
16415 });
16416 runner.state.merge_repairs.push(request);
16417 runner.begin_merge_repair_review(0).unwrap();
16418 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16419 assert!(!runner.state.gate_status().ok());
16420 assert!(runner.state.land_approval.is_none());
16421 let disk = RunState::load(&runner.state.id).unwrap();
16422 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16423 assert!(disk.merge_repairs[0].gate[0].ok());
16424 assert_eq!(
16425 disk.merge_repairs[0].phase,
16426 crate::run::MergeRepairPhase::Reviewing
16427 );
16428 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16429 assert!(disk.reviews.is_empty());
16430 }
16431
16432 #[tokio::test]
16433 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16434 crate::run::pin_test_home();
16435 for phase in [
16436 crate::run::MergeRepairPhase::Fixing,
16437 crate::run::MergeRepairPhase::Requested,
16438 ] {
16439 let mut runner = runner_at(RunStatus::Landing);
16440 runner.state.config.graph.land_rounds = 0;
16441 runner.state.merge_repairs.push(merge_repair_request(phase));
16442 runner.state.save().unwrap();
16443 assert!(!runner.repair_merge_review().await.unwrap());
16444 assert_eq!(runner.state.status, RunStatus::Blocked);
16445 assert_eq!(
16446 runner.state.merge_repairs[0].phase,
16447 crate::run::MergeRepairPhase::Failed
16448 );
16449 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16450 assert!(
16451 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16452 "interrupted"
16453 } else {
16454 "budget exhausted"
16455 })
16456 );
16457 assert!(runner.state.merge.is_none());
16458 }
16459 }
16460
16461 #[test]
16465 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16466 crate::run::pin_test_home();
16467 let mut runner = runner_at(RunStatus::Implementing);
16468 let interrupt = Pause::new();
16469 runner.watch_interrupt(interrupt.clone());
16470
16471 interrupt.park_because("task a1b2 asked to run first");
16472
16473 assert!(runner.park_here().expect("park_here"));
16474 assert!(runner.state.parked);
16475 let last = runner.state.events.last().expect("a park event");
16476 assert_eq!(last.node, "park");
16477 assert!(
16478 last.message.contains("task a1b2 asked to run first"),
16479 "expected the interrupt reason in {:?}",
16480 last.message
16481 );
16482 }
16483
16484 #[test]
16487 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16488 crate::run::pin_test_home();
16489 let mut runner = runner_at(RunStatus::Implementing);
16490 runner.state.begin_pass();
16491 let shutdown = Pause::new();
16492 runner.on_pause(shutdown.clone());
16493 let upgrade = ParkCause::Upgrade {
16494 from: "0.121.0".to_owned(),
16495 to: Some("0.121.1".to_owned()),
16496 };
16497 shutdown.park_for(upgrade.clone());
16498 shutdown.park_for(ParkCause::Operator);
16499
16500 assert!(runner.park_here().expect("park_here"));
16501 assert_eq!(
16502 runner.state.passes[0].end,
16503 Some(crate::run::PassEnd::Parked { cause: upgrade })
16504 );
16505 let last = runner.state.events.last().expect("a park event");
16506 assert!(last.message.contains("0.121.0"), "{}", last.message);
16507 }
16508
16509 #[tokio::test]
16510 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16511 crate::run::pin_test_home();
16512 let mut runner = runner_at(RunStatus::Reviewing);
16513 runner.state.save().unwrap();
16514 let mut execution = runner.claim_driver().await.expect("first execution");
16515 assert_eq!(execution.runner.state.passes.len(), 1);
16516 execution
16517 .driver
16518 .finish(&mut execution.runner.state)
16519 .await
16520 .unwrap();
16521 }
16522
16523 #[tokio::test]
16524 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
16525 crate::run::pin_test_home();
16526 let mut runner = runner_at(RunStatus::Reviewing);
16527 runner.state.config.agents = vec![conductor()];
16528 runner.state.config.graph.timeout_resource_wait = 0;
16529 runner.state.save().unwrap();
16530 let id = runner.state.id.clone();
16531 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16532 assert!(runner.claim_driver().await.is_err());
16533 drop(owner);
16534 drop(runner);
16535 let mut resumed = Runner::resume(&id).unwrap();
16536 assert_eq!(resumed.state.driver_claimed, Some(false));
16537 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
16538 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
16539 let mut execution = resumed
16540 .claim_driver()
16541 .await
16542 .expect("retry initial claim after reload");
16543 assert_eq!(execution.runner.state.driver_claimed, Some(true));
16544 assert_eq!(execution.runner.state.passes.len(), 1);
16545 execution
16546 .driver
16547 .finish(&mut execution.runner.state)
16548 .await
16549 .unwrap();
16550 }
16551
16552 #[tokio::test]
16553 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
16554 crate::run::pin_test_home();
16555 let mut runner = runner_at(RunStatus::Reviewing);
16556 runner.state.config.agents = vec![conductor()];
16557 runner.state.save().unwrap();
16558 let id = runner.state.id.clone();
16559 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
16560 let mut resumed = Runner::resume(&id).unwrap();
16561 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
16562 assert!(resumed.claim_driver().await.is_err());
16563 assert!(RunState::load(&id).unwrap().passes.is_empty());
16564 drop(lease);
16565 let mut execution = resumed.claim_driver().await.unwrap();
16566 execution
16567 .driver
16568 .finish(&mut execution.runner.state)
16569 .await
16570 .unwrap();
16571 }
16572
16573 #[tokio::test]
16574 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
16575 crate::run::pin_test_home();
16576 let mut runner = runner_at(RunStatus::Reviewing);
16577 let state = &mut runner.state;
16578 state.config.agents = vec![conductor()];
16579 state.driver_claimed = None;
16580 state.driver_pid = Some(std::process::id());
16581 state.driver_started_at = None;
16582 state.begin_pass();
16583 state.save().unwrap();
16584 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
16585 std::fs::write(
16586 state.dir().join("driver.json"),
16587 serde_json::to_vec(&serde_json::json!({
16588 "pid": state.driver_pid, "started_at": null,
16589 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
16590 }))
16591 .unwrap(),
16592 )
16593 .unwrap();
16594 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
16595 drop(lease);
16596 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
16597 let mut resumed = Runner::resume(&state.id).unwrap();
16598 assert!(!resumed.state.driver_exited);
16599 let mut execution = resumed
16600 .claim_driver()
16601 .await
16602 .expect("lease proves previous execution ended");
16603 assert_eq!(execution.runner.state.passes.len(), 2);
16604 execution
16605 .driver
16606 .finish(&mut execution.runner.state)
16607 .await
16608 .unwrap();
16609 }
16610
16611 #[tokio::test]
16612 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16613 crate::run::pin_test_home();
16614 let mut runner = runner_at(RunStatus::Reviewing);
16615 let mut json = serde_json::to_value(&runner.state).unwrap();
16616 json.as_object_mut().unwrap().remove("driver_claimed");
16617 runner.state = serde_json::from_value(json).unwrap();
16618 assert_eq!(runner.state.driver_claimed, None);
16619 runner.state.save().unwrap();
16620 let mut execution = runner
16621 .claim_driver()
16622 .await
16623 .expect("the execution lease, not a missing marker, excludes a second driver");
16624 assert_eq!(execution.runner.state.passes.len(), 1);
16625 execution
16626 .driver
16627 .finish(&mut execution.runner.state)
16628 .await
16629 .unwrap();
16630 }
16631
16632 #[tokio::test]
16633 async fn driver_admission_refuses_a_proven_live_driver() {
16634 crate::run::pin_test_home();
16635 let mut runner = runner_at(RunStatus::Reviewing);
16636 let pid = std::process::id();
16637 runner.state.driver_pid = Some(pid);
16638 runner.state.driver_started_at = crate::proc::process_started_at(pid);
16639 assert!(runner.state.driver_started_at.is_some());
16640 runner.state.save().unwrap();
16641 let error = runner
16642 .claim_driver()
16643 .await
16644 .err()
16645 .expect("a live driver must refuse");
16646 assert!(error.to_string().contains("live driver"), "{error:#}");
16647 let saved = RunState::load(&runner.state.id).unwrap();
16648 assert_eq!(saved.driver_pid, Some(pid));
16649 assert_eq!(saved.status, RunStatus::Reviewing);
16650 assert!(saved.passes.is_empty());
16651 }
16652
16653 #[test]
16654 fn parking_resource_halt_persists_the_pass_once() {
16655 crate::run::pin_test_home();
16656 let mut runner = runner_at(RunStatus::Reviewing);
16657 runner.state.begin_pass();
16658 runner.state.resource_block = Some(crate::run::ResourceBlock {
16659 node: "fix".into(),
16660 kind: "shared build cache".into(),
16661 detail: "cancelled".into(),
16662 at: Timestamp::now(),
16663 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16664 });
16665 let cause = ParkCause::Upgrade {
16666 from: "old".into(),
16667 to: Some("new".into()),
16668 };
16669 runner.pause.park_for(cause.clone());
16670 runner.stop_for_resource().unwrap();
16673 assert!(runner.resource_halt().unwrap());
16674 assert!(runner.park_here().unwrap());
16675 let saved = RunState::load(&runner.state.id).unwrap();
16676 assert!(saved.parked);
16677 assert_eq!(saved.status, RunStatus::Reviewing);
16678 assert!(saved.passes[0].ended_at.is_some());
16679 assert_eq!(
16680 saved.passes[0].end,
16681 Some(crate::run::PassEnd::Parked {
16682 cause: cause.clone()
16683 })
16684 );
16685 assert_eq!(
16686 saved.resource_block.unwrap().cause,
16687 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16688 );
16689 assert_eq!(
16690 saved
16691 .events
16692 .iter()
16693 .filter(|event| event.node == "park")
16694 .count(),
16695 1
16696 );
16697 }
16698
16699 #[test]
16707 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16708 crate::run::pin_test_home();
16709 let mut runner = runner_at(RunStatus::Implementing);
16710 let shutdown = Pause::new();
16711 runner.on_pause(shutdown.clone());
16712 let interrupt = Pause::new();
16713 runner.watch_interrupt(interrupt.clone());
16714
16715 assert!(!runner.park_here().expect("park_here"));
16717 assert!(!runner.state.parked);
16718
16719 interrupt.park_because("test");
16721 assert!(!shutdown.parked());
16722 assert!(runner.park_here().expect("park_here"));
16723 }
16724
16725 #[tokio::test]
16739 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16740 crate::run::pin_test_home();
16741 let mut runner = runner_at(RunStatus::Implementing);
16742 let interrupt = Pause::new();
16743 runner.watch_interrupt(interrupt.clone());
16744
16745 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16746 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16747
16748 let node = async move {
16752 started_tx.send(()).expect("send started");
16753 finish_rx.await.expect("recv finish");
16754 "node finished"
16755 };
16756
16757 let interrupter = async move {
16758 started_rx.await.expect("recv started");
16759 interrupt.park_because("higher-priority task waiting");
16761 tokio::task::yield_now().await;
16765 finish_tx.send(()).expect("send finish");
16766 };
16767
16768 let (node_result, ()) = tokio::join!(node, interrupter);
16769 assert_eq!(
16770 node_result, "node finished",
16771 "the in-flight call ran to completion"
16772 );
16773
16774 assert!(runner.park_here().expect("park_here"));
16777 assert!(runner.state.parked);
16778 }
16779
16780 #[test]
16786 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16787 crate::run::pin_test_home();
16788 let mut runner = runner_at(RunStatus::Judging);
16789 runner.state.config.agents = vec![conductor()];
16793 runner.state.candidates = vec![Candidate {
16794 index: 0,
16795 label: 'A',
16796 agent: "alpha".to_owned(),
16797 branch: "magi/x/A".to_owned(),
16798 worktree: PathBuf::from("/nonexistent/worktree"),
16799 summary: "did the thing".to_owned(),
16800 stat: "1 file changed".to_owned(),
16801 files: 1,
16802 commits: 1,
16803 empty: false,
16804 failed: None,
16805 verified_noop: None,
16806 duration_ms: 1234,
16807 folded: false,
16808 }];
16809 let run_id = runner.state.id.clone();
16810
16811 let interrupt = Pause::new();
16812 runner.watch_interrupt(interrupt.clone());
16813 interrupt.park_because("task c3d4 asked to run first");
16814 assert!(runner.park_here().expect("park_here"));
16815
16816 let resumed = Runner::resume(&run_id).expect("resume");
16817 assert_eq!(resumed.state.candidates.len(), 1);
16818 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16819 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16820 assert_eq!(resumed.state.status, runner.state.status);
16821 assert!(
16822 resumed.state.parked,
16823 "still parked until `execute` actually walks the graph again"
16824 );
16825 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16826 }
16827
16828 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16830 let mut q = ask::Question::new(
16831 run.to_owned(),
16832 "implement".to_owned(),
16833 "impl-A".to_owned(),
16834 "Which storage backend should the cache use?".to_owned(),
16835 String::new(),
16836 vec!["SQLite".to_owned(), "Redis".to_owned()],
16837 );
16838 store.put(&mut q).unwrap();
16839 q
16840 }
16841
16842 #[test]
16843 fn a_failed_runs_open_question_is_abandoned() {
16844 ask_test_home();
16845 let store = ask::Questions::open();
16846 let mut runner = runner_at(RunStatus::Failed);
16847 let run = runner.state.id.clone();
16848 let q = ask_open_question(&store, &run);
16849
16850 runner.settle_questions();
16851
16852 let back = store.get(&q.id).unwrap();
16853 assert!(
16854 !back.status.open(),
16855 "the seat that asked died with the run; nobody is left to read an answer"
16856 );
16857 assert!(
16858 back.detail.contains(&run) && back.detail.contains("failed"),
16859 "the reason names what the run became, not just that it is gone: {}",
16860 back.detail
16861 );
16862 }
16863
16864 #[test]
16865 fn a_merged_runs_open_question_is_abandoned_too() {
16866 ask_test_home();
16867 let store = ask::Questions::open();
16868 for status in [RunStatus::Merged, RunStatus::Ready] {
16871 let mut runner = runner_at(status);
16872 let run = runner.state.id.clone();
16873 let q = ask_open_question(&store, &run);
16874
16875 runner.settle_questions();
16876
16877 let back = store.get(&q.id).unwrap();
16878 assert!(
16879 !back.status.open(),
16880 "{status:?} run's question must not outlive the run"
16881 );
16882 }
16883 }
16884
16885 #[test]
16886 fn a_still_resumable_runs_open_question_is_left_alone() {
16887 ask_test_home();
16888 let store = ask::Questions::open();
16889 for status in [RunStatus::Blocked, RunStatus::Stalled] {
16895 let mut runner = runner_at(status);
16896 let run = runner.state.id.clone();
16897 let q = ask_open_question(&store, &run);
16898
16899 runner.settle_questions();
16900
16901 let back = store.get(&q.id).unwrap();
16902 assert!(
16903 back.status.open(),
16904 "{status:?} is still alive; the question must still be waiting"
16905 );
16906 }
16907 }
16908
16909 #[test]
16910 fn settle_questions_never_touches_an_already_answered_question() {
16911 ask_test_home();
16912 let store = ask::Questions::open();
16913 let mut runner = runner_at(RunStatus::Failed);
16914 let run = runner.state.id.clone();
16915 let mut q = ask_open_question(&store, &run);
16916 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16917 .unwrap();
16918 store.put(&mut q).unwrap();
16919
16920 runner.settle_questions();
16925 runner.settle_questions();
16926
16927 let back = store.get(&q.id).unwrap();
16928 assert_eq!(
16929 back.status,
16930 ask::QuestionStatus::Answered,
16931 "a real answer is a decision on record, never overwritten by a sweep"
16932 );
16933 }
16934
16935 #[tokio::test]
16946 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16947 crate::run::pin_test_home();
16948 let tmp = tempfile::tempdir().expect("tempdir");
16949 let repo = tmp.path().join("repo");
16950 std::fs::create_dir_all(&repo).unwrap();
16951 init_repo(&repo);
16952
16953 let mut config = Config::default();
16954 config.graph.worktree_root = Some(tmp.path().join("wt"));
16955
16956 let mut state = RunState::new(
16957 repo.clone(),
16958 "main".to_owned(),
16959 "deadbeef".to_owned(),
16960 "task".to_owned(),
16961 config,
16962 );
16963 let root = state.worktree_root();
16964 let wt_a = root.join("cand-A");
16965 let wt_b = root.join("cand-B");
16966 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16967 .await
16968 .expect("worktree A");
16969 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16970 .await
16971 .expect("worktree B");
16972
16973 state.candidates = vec![
16974 Candidate {
16975 index: 0,
16976 label: 'A',
16977 agent: "alpha".to_owned(),
16978 branch: "magi/x/A".to_owned(),
16979 worktree: wt_a.clone(),
16980 summary: String::new(),
16981 stat: String::new(),
16982 files: 0,
16983 commits: 0,
16984 empty: false,
16985 failed: None,
16986 verified_noop: None,
16987 duration_ms: 0,
16988 folded: false,
16989 },
16990 Candidate {
16991 index: 1,
16992 label: 'B',
16993 agent: "beta".to_owned(),
16994 branch: "magi/x/B".to_owned(),
16995 worktree: wt_b.clone(),
16996 summary: String::new(),
16997 stat: String::new(),
16998 files: 0,
16999 commits: 0,
17000 empty: false,
17001 failed: None,
17002 verified_noop: None,
17003 duration_ms: 0,
17004 folded: false,
17005 },
17006 ];
17007 state.tally = Some(Tally {
17008 first_choice: BTreeMap::from([('A', 1)]),
17009 borda: BTreeMap::new(),
17010 winner: 'A',
17011 rankings: 1,
17012 unanimous_initial: true,
17013 deliberated: false,
17014 changed_votes: 0,
17015 unanimous_final: true,
17016 tie_break: None,
17017 judges: 1,
17018 present: 1,
17019 quorum: 1,
17020 met_quorum: true,
17021 uncontested: None,
17022 });
17023 state.status = RunStatus::Ready;
17024
17025 fold_run(&mut state, false, &crate::run::home())
17026 .await
17027 .expect("fold_run");
17028
17029 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17030 assert!(
17031 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17032 "the unmerged winner's branch survives"
17033 );
17034 assert!(
17035 !state.candidates[0].folded,
17036 "the winner is not marked folded"
17037 );
17038
17039 assert!(!wt_b.exists(), "the loser's worktree is removed");
17040 assert!(
17041 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17042 "the loser's branch is removed"
17043 );
17044 assert!(state.candidates[1].folded, "the loser is marked folded");
17045 }
17046
17047 #[tokio::test]
17048 #[cfg(unix)]
17049 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17050 use std::os::fd::AsRawFd as _;
17051 use std::time::Duration;
17052
17053 let tmp = tempfile::tempdir().unwrap();
17054 let repo = tmp.path().join("repo");
17055 std::fs::create_dir_all(&repo).unwrap();
17056 init_repo(&repo);
17057 let common = repo.join(".git");
17058 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17059 let home = tmp.path().join("home");
17060 let mut config = Config::default();
17061 config.coordination = crate::config::test_coordination();
17062 config.graph.worktree_root = Some(tmp.path().join("wt"));
17063 config.graph.timeout_resource_wait = 1;
17064 let mut state = RunState::new(
17065 repo.clone(),
17066 "main".into(),
17067 "deadbeef".into(),
17068 "task".into(),
17069 config,
17070 );
17071 let root = state.worktree_root();
17072 let candidate = root.join("cand-A");
17073 let reviewer = root.join("review-1");
17074 let branch = state.branch_for('A');
17075 git::worktree_add_branch(&repo, &candidate, &branch, "main")
17076 .await
17077 .unwrap();
17078 git::worktree_add_detached(&repo, &reviewer, "main")
17079 .await
17080 .unwrap();
17081 git::acquire_worktree_config(&repo).await.unwrap();
17082 state.enabled_worktree_config = true;
17083 state.status = RunStatus::Ready;
17084 state.candidates.push(Candidate {
17085 index: 0,
17086 label: 'A',
17087 agent: "implement".into(),
17088 branch: branch.clone(),
17089 worktree: candidate.clone(),
17090 summary: String::new(),
17091 stat: String::new(),
17092 files: 0,
17093 commits: 0,
17094 empty: false,
17095 failed: None,
17096 verified_noop: None,
17097 duration_ms: 0,
17098 folded: false,
17099 });
17100 state.save_under(&home).unwrap();
17101 let config_before = std::fs::read(common.join("config")).unwrap();
17102 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17103 .await
17104 .unwrap();
17105 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17108 assert_eq!(
17109 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17110 0
17111 );
17112 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17113 assert!(
17114 tokio::time::timeout(Duration::from_millis(200), &mut fold)
17115 .await
17116 .is_err()
17117 );
17118 assert!(candidate.exists());
17119 assert!(reviewer.exists());
17120 assert!(git::branch_exists(&repo, &branch).await.unwrap());
17121 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17122 assert_eq!(
17123 git::git(&repo, &["worktree", "list", "--porcelain"])
17124 .await
17125 .unwrap(),
17126 worktrees_before
17127 );
17128 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17129 .await
17130 .unwrap()
17131 .unwrap_err();
17132 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17133 drop(fold);
17134 assert!(candidate.exists() && reviewer.exists());
17135 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17136 assert_eq!(
17137 state.resource_block.as_ref().unwrap().kind,
17138 "repository coordination"
17139 );
17140 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17141 drop(external);
17142 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17143 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17144 .await
17145 .unwrap()
17146 .unwrap();
17147 drop(fold);
17148 assert!(removed.contains(&branch));
17149 assert!(!root.exists());
17150 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17151 assert!(state.candidates[0].folded);
17152 assert!(!state.enabled_worktree_config);
17153 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17154 let enabled = git::git_raw(
17155 &repo,
17156 &["config", "--local", "--get", "extensions.worktreeConfig"],
17157 )
17158 .await
17159 .unwrap();
17160 assert!(!enabled.ok(), "the last config reference was released");
17161 }
17162
17163 #[tokio::test]
17166 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17167 let tmp = tempfile::tempdir().expect("tempdir");
17168 let repo = tmp.path().join("repo");
17169 std::fs::create_dir_all(&repo).unwrap();
17170 init_repo(&repo);
17171 let home = tmp.path().join("home");
17172
17173 let mut config = Config::default();
17174 config.graph.worktree_root = Some(tmp.path().join("wt"));
17175 let mut state = RunState::new(
17176 repo.clone(),
17177 "main".to_owned(),
17178 "deadbeef".to_owned(),
17179 "task".to_owned(),
17180 config,
17181 );
17182 git::git(&repo, &["branch", "magi/x/A", "main"])
17184 .await
17185 .expect("branch");
17186 state.candidates = vec![Candidate {
17187 index: 0,
17188 label: 'A',
17189 agent: "alpha".to_owned(),
17190 branch: "magi/x/A".to_owned(),
17191 worktree: state.worktree_root().join("cand-A"),
17192 summary: String::new(),
17193 stat: String::new(),
17194 files: 0,
17195 commits: 0,
17196 empty: false,
17197 failed: None,
17198 verified_noop: None,
17199 duration_ms: 0,
17200 folded: true,
17201 }];
17202 state.released_to = Some("20260901-000000-new1".to_owned());
17203 state.released_branches = vec!["magi/x/A".to_owned()];
17204
17205 fold_run(&mut state, true, &home).await.expect("fold_run");
17206 state.status = RunStatus::Merged;
17207 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17208 state.merge = Some(crate::run::MergeOutcome {
17209 mode: MergeMode::Pr,
17210 ok: true,
17211 detail: "confirmed merged".into(),
17212 empty: false,
17213 });
17214 fold_run(&mut state, true, &home)
17215 .await
17216 .expect("merged fold");
17217
17218 assert!(
17219 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17220 "the handed-over branch survives a fold"
17221 );
17222 }
17223
17224 #[tokio::test]
17228 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17229 let tmp = tempfile::tempdir().expect("tempdir");
17230 let repo = tmp.path().join("repo");
17231 std::fs::create_dir_all(&repo).unwrap();
17232 init_repo(&repo);
17233 let run = |args: &[&str]| {
17234 let out = std::process::Command::new("git")
17235 .quiet()
17236 .args(args)
17237 .current_dir(&repo)
17238 .output()
17239 .expect("spawn git");
17240 assert!(out.status.success(), "git {args:?}");
17241 };
17242 run(&["branch", "magi/x/A"]);
17243 run(&["checkout", "-q", "-b", "magi/x/B"]);
17244 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17245 run(&["add", "-A"]);
17246 run(&["commit", "-q", "-m", "work"]);
17247 run(&["checkout", "-q", "main"]);
17248
17249 let probe = RunState::new(
17253 repo.clone(),
17254 "main".to_owned(),
17255 "deadbeef".to_owned(),
17256 "task".to_owned(),
17257 Config::default(),
17258 );
17259 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17260 let bare = tmp.path().join("origin.git");
17261 let out = std::process::Command::new("git")
17262 .quiet()
17263 .args(["init", "-q", "--bare"])
17264 .arg(&bare)
17265 .output()
17266 .expect("spawn git");
17267 assert!(out.status.success(), "git init --bare");
17268 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17269 run(&["push", "-q", "origin", "main"]);
17270
17271 let mut state = RunState::new(
17272 repo.clone(),
17273 "main".to_owned(),
17274 "deadbeef".to_owned(),
17275 "task".to_owned(),
17276 Config::default(),
17277 );
17278 state.seeds = vec![refs::Seed {
17279 token: "magi/27b2/A".to_owned(),
17280 kind: refs::SeedKind::Unresolved,
17281 sha: String::new(),
17282 branch: true,
17283 detail: "no branch or commit named magi/27b2/A".to_owned(),
17284 }];
17285
17286 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17287 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17288 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17289 let detail = empty_candidate_detail(&state, "main");
17290 assert!(detail.starts_with("empty candidate"), "{detail}");
17291 assert!(!detail.contains("magi/27b2/A"), "{detail}");
17292 assert!(detail.contains("--start-from"), "{detail}");
17293 }
17294
17295 #[tokio::test]
17304 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17305 let tmp = tempfile::tempdir().expect("tempdir");
17306 let repo = tmp.path().join("repo");
17307 std::fs::create_dir_all(&repo).unwrap();
17308 init_repo(&repo);
17309
17310 let mut config = Config::default();
17311 config.merge.mode = MergeMode::Local;
17312
17313 let mut state = RunState::new(
17314 repo.clone(),
17315 "main".to_owned(),
17316 "deadbeef".to_owned(),
17317 "task".to_owned(),
17318 config,
17319 );
17320 state.candidates = vec![Candidate {
17321 index: 0,
17322 label: 'A',
17323 agent: "alpha".to_owned(),
17324 branch: "does-not-exist".to_owned(),
17325 worktree: repo.clone(),
17326 summary: String::new(),
17327 stat: String::new(),
17328 files: 0,
17329 commits: 0,
17330 empty: false,
17331 failed: None,
17332 verified_noop: None,
17333 duration_ms: 0,
17334 folded: false,
17335 }];
17336 state.tally = Some(Tally {
17337 first_choice: BTreeMap::from([('A', 1)]),
17338 borda: BTreeMap::new(),
17339 winner: 'A',
17340 rankings: 1,
17341 unanimous_initial: true,
17342 deliberated: false,
17343 changed_votes: 0,
17344 unanimous_final: true,
17345 tie_break: None,
17346 judges: 0,
17347 present: 0,
17348 quorum: 0,
17349 met_quorum: true,
17350 uncontested: Some("only candidate A produced a change".to_owned()),
17351 });
17352 state.reviews = vec![ReviewRound {
17353 round: 1,
17354 head: "deadbeef".to_owned(),
17355 verified_head: None,
17356 verified_at: None,
17357 reviews: Vec::new(),
17358 e2e: Vec::new(),
17359 fix: None,
17360 blocking: 0,
17361 answered: 0,
17362 expected: 0,
17363 clean: true,
17364 verify_retried: false,
17365 e2e_deferred: false,
17366 e2e_defer_reason: None,
17367 progressed: false,
17368 vote_split: false,
17369 reconsideration: Vec::new(),
17370 verdict: None,
17371 }];
17372 state.gate = vec![CommandOutcome {
17373 command: "test".to_owned(),
17374 code: Some(0),
17375 output_tail: String::new(),
17376 duration_ms: 0,
17377 resource_blocked: false,
17378 }];
17379 state.gate_ran = true;
17380 state.status = RunStatus::Ready;
17385 state.merge = Some(MergeOutcome {
17386 mode: MergeMode::Local,
17387 ok: false,
17388 detail: "already concluded".to_owned(),
17389 empty: false,
17390 });
17391
17392 let mut runner = Runner {
17393 state,
17394 roles: ResolvedRoles {
17395 implementers: Vec::new(),
17396 judges: Vec::new(),
17397 reviewers: Vec::new(),
17398 fixer: None,
17399 conductor: conductor(),
17400 implementer_roster: Vec::new(),
17401 judge_roster: Vec::new(),
17402 reviewer_roster: Vec::new(),
17403 },
17404 sem: Arc::new(Semaphore::new(1)),
17405 pause: Pause::new(),
17406 interrupt: Pause::new(),
17407 };
17408
17409 runner.merge().await.expect("merge");
17410
17411 assert_eq!(
17412 runner.state.status,
17413 RunStatus::Ready,
17414 "a concluded run's status must not change on reentry"
17415 );
17416 assert_eq!(
17417 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17418 Some("already concluded"),
17419 "merge must not run again once the node already recorded an outcome"
17420 );
17421 }
17422
17423 #[tokio::test]
17432 async fn merge_refuses_a_gate_that_has_not_actually_run() {
17433 let tmp = tempfile::tempdir().expect("tempdir");
17434 let repo = tmp.path().join("repo");
17435 std::fs::create_dir_all(&repo).unwrap();
17436 init_repo(&repo);
17437
17438 let mut config = Config::default();
17439 config.merge.mode = MergeMode::Local;
17440
17441 let mut state = RunState::new(
17442 repo.clone(),
17443 "main".to_owned(),
17444 "deadbeef".to_owned(),
17445 "task".to_owned(),
17446 config,
17447 );
17448 state.candidates = vec![Candidate {
17449 index: 0,
17450 label: 'A',
17451 agent: "alpha".to_owned(),
17452 branch: "does-not-exist".to_owned(),
17453 worktree: repo.clone(),
17454 summary: String::new(),
17455 stat: String::new(),
17456 files: 0,
17457 commits: 0,
17458 empty: false,
17459 failed: None,
17460 verified_noop: None,
17461 duration_ms: 0,
17462 folded: false,
17463 }];
17464 state.tally = Some(Tally {
17465 first_choice: BTreeMap::from([('A', 1)]),
17466 borda: BTreeMap::new(),
17467 winner: 'A',
17468 rankings: 1,
17469 unanimous_initial: true,
17470 deliberated: false,
17471 changed_votes: 0,
17472 unanimous_final: true,
17473 tie_break: None,
17474 judges: 0,
17475 present: 0,
17476 quorum: 0,
17477 met_quorum: true,
17478 uncontested: Some("only candidate A produced a change".to_owned()),
17479 });
17480 state.reviews = vec![ReviewRound {
17481 round: 1,
17482 head: "deadbeef".to_owned(),
17483 verified_head: None,
17484 verified_at: None,
17485 reviews: Vec::new(),
17486 e2e: Vec::new(),
17487 fix: None,
17488 blocking: 0,
17489 answered: 0,
17490 expected: 0,
17491 clean: true,
17492 verify_retried: false,
17493 e2e_deferred: false,
17494 e2e_defer_reason: None,
17495 progressed: false,
17496 vote_split: false,
17497 reconsideration: Vec::new(),
17498 verdict: None,
17499 }];
17500 state.gate = Vec::new();
17502 state.gate_ran = false;
17503 state.status = RunStatus::Gating;
17504
17505 let mut runner = Runner {
17506 state,
17507 roles: ResolvedRoles {
17508 implementers: Vec::new(),
17509 judges: Vec::new(),
17510 reviewers: Vec::new(),
17511 fixer: None,
17512 conductor: conductor(),
17513 implementer_roster: Vec::new(),
17514 judge_roster: Vec::new(),
17515 reviewer_roster: Vec::new(),
17516 },
17517 sem: Arc::new(Semaphore::new(1)),
17518 pause: Pause::new(),
17519 interrupt: Pause::new(),
17520 };
17521
17522 runner.merge().await.expect("merge");
17523
17524 assert!(
17525 runner.state.merge.is_none(),
17526 "an empty gate must never be read as a passing one: {:?}",
17527 runner.state.merge
17528 );
17529 }
17530
17531 #[tokio::test]
17538 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
17539 let tmp = tempfile::tempdir().expect("tempdir");
17540 let repo = tmp.path().join("repo");
17541 std::fs::create_dir_all(&repo).unwrap();
17542 init_repo(&repo);
17543
17544 let config = Config::default();
17546
17547 let mut state = RunState::new(
17548 repo.clone(),
17549 "main".to_owned(),
17550 "deadbeef".to_owned(),
17551 "task".to_owned(),
17552 config,
17553 );
17554 state.candidates = vec![Candidate {
17555 index: 0,
17556 label: 'A',
17557 agent: "alpha".to_owned(),
17558 branch: "does-not-exist".to_owned(),
17559 worktree: repo.clone(),
17560 summary: String::new(),
17561 stat: String::new(),
17562 files: 0,
17563 commits: 0,
17564 empty: false,
17565 failed: None,
17566 verified_noop: None,
17567 duration_ms: 0,
17568 folded: false,
17569 }];
17570 state.tally = Some(Tally {
17571 first_choice: BTreeMap::from([('A', 1)]),
17572 borda: BTreeMap::new(),
17573 winner: 'A',
17574 rankings: 1,
17575 unanimous_initial: true,
17576 deliberated: false,
17577 changed_votes: 0,
17578 unanimous_final: true,
17579 tie_break: None,
17580 judges: 0,
17581 present: 0,
17582 quorum: 0,
17583 met_quorum: true,
17584 uncontested: Some("only candidate A produced a change".to_owned()),
17585 });
17586 state.reviews = vec![ReviewRound {
17587 round: 1,
17588 head: "deadbeef".to_owned(),
17589 verified_head: None,
17590 verified_at: None,
17591 reviews: Vec::new(),
17592 e2e: Vec::new(),
17593 fix: None,
17594 blocking: 0,
17595 answered: 0,
17596 expected: 0,
17597 clean: true,
17598 verify_retried: false,
17599 e2e_deferred: false,
17600 e2e_defer_reason: None,
17601 progressed: false,
17602 vote_split: false,
17603 reconsideration: Vec::new(),
17604 verdict: None,
17605 }];
17606
17607 let mut runner = Runner {
17608 state,
17609 roles: ResolvedRoles {
17610 implementers: Vec::new(),
17611 judges: Vec::new(),
17612 reviewers: Vec::new(),
17613 fixer: None,
17614 conductor: conductor(),
17615 implementer_roster: Vec::new(),
17616 judge_roster: Vec::new(),
17617 reviewer_roster: Vec::new(),
17618 },
17619 sem: Arc::new(Semaphore::new(1)),
17620 pause: Pause::new(),
17621 interrupt: Pause::new(),
17622 };
17623
17624 runner.gate().await.expect("gate");
17625 assert!(
17626 runner.state.gate_ran,
17627 "zero configured commands is still a real attempt, not an unrun gate"
17628 );
17629 assert!(runner.state.gate.is_empty());
17630 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17631 assert_ne!(
17632 runner.state.status,
17633 RunStatus::Blocked,
17634 "a gate with nothing to check must not read as failed"
17635 );
17636
17637 runner.merge().await.expect("merge");
17638 assert_eq!(
17639 runner.state.status,
17640 RunStatus::Ready,
17641 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17642 );
17643 }
17644
17645 #[tokio::test]
17656 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17657 crate::run::pin_test_home();
17658 let home = crate::run::home();
17659
17660 let tmp = tempfile::tempdir().expect("tempdir");
17661 let repo = tmp.path().join("repo");
17662 std::fs::create_dir_all(&repo).unwrap();
17663 init_repo(&repo);
17664 let cache_dir = tmp.path().join("target");
17667
17668 let mut config = Config::default();
17669 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17670 config.graph.timeout_verify = Some(2);
17673 config.graph.timeout_resource_wait = 2;
17674
17675 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17676 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17677 .expect("no io error acquiring directly")
17678 {
17679 crate::cache::AcquireOutcome::Acquired(g) => g,
17680 crate::cache::AcquireOutcome::Busy(b) => {
17681 panic!("expected the direct acquire to win the lease first: {b:?}")
17682 }
17683 };
17684
17685 let mut state = RunState::new(
17686 repo.clone(),
17687 "main".to_owned(),
17688 "deadbeef".to_owned(),
17689 "task".to_owned(),
17690 config,
17691 );
17692 state.candidates = vec![Candidate {
17693 index: 0,
17694 label: 'A',
17695 agent: "alpha".to_owned(),
17696 branch: "does-not-exist".to_owned(),
17697 worktree: repo.clone(),
17698 summary: String::new(),
17699 stat: String::new(),
17700 files: 0,
17701 commits: 0,
17702 empty: false,
17703 failed: None,
17704 verified_noop: None,
17705 duration_ms: 0,
17706 folded: false,
17707 }];
17708 state.tally = Some(Tally {
17709 first_choice: BTreeMap::from([('A', 1)]),
17710 borda: BTreeMap::new(),
17711 winner: 'A',
17712 rankings: 1,
17713 unanimous_initial: true,
17714 deliberated: false,
17715 changed_votes: 0,
17716 unanimous_final: true,
17717 tie_break: None,
17718 judges: 0,
17719 present: 0,
17720 quorum: 0,
17721 met_quorum: true,
17722 uncontested: Some("only candidate A produced a change".to_owned()),
17723 });
17724 state.reviews = vec![ReviewRound {
17725 round: 1,
17726 head: "deadbeef".to_owned(),
17727 verified_head: None,
17728 verified_at: None,
17729 reviews: Vec::new(),
17730 e2e: Vec::new(),
17731 fix: None,
17732 blocking: 0,
17733 answered: 0,
17734 expected: 0,
17735 clean: true,
17736 verify_retried: false,
17737 e2e_deferred: false,
17738 e2e_defer_reason: None,
17739 progressed: false,
17740 vote_split: false,
17741 reconsideration: Vec::new(),
17742 verdict: None,
17743 }];
17744
17745 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
17746 state.reviews[0].e2e = vec![CommandOutcome {
17747 command: state.config.verify.e2e[0].clone(),
17748 code: Some(0),
17749 output_tail: String::new(),
17750 duration_ms: 0,
17751 resource_blocked: false,
17752 }];
17753 let mut runner = Runner {
17754 state,
17755 roles: ResolvedRoles {
17756 implementers: Vec::new(),
17757 judges: Vec::new(),
17758 reviewers: Vec::new(),
17759 fixer: None,
17760 conductor: conductor(),
17761 implementer_roster: Vec::new(),
17762 judge_roster: Vec::new(),
17763 reviewer_roster: Vec::new(),
17764 },
17765 sem: Arc::new(Semaphore::new(1)),
17766 pause: Pause::new(),
17767 interrupt: Pause::new(),
17768 };
17769
17770 let started = std::time::Instant::now();
17771 runner.gate().await.expect("gate");
17772 assert!(
17773 started.elapsed() < Duration::from_secs(1),
17774 "a gate with nothing to run must never wait on a lease it never needed"
17775 );
17776 assert!(
17777 runner.state.gate_ran,
17778 "zero commands is still a real, immediate attempt"
17779 );
17780 assert!(runner.state.gate.is_empty());
17781 assert_ne!(
17782 runner.state.status,
17783 RunStatus::Blocked,
17784 "must not read as resource-blocked on a lease it never asked for"
17785 );
17786 }
17787
17788 #[cfg(unix)]
17791 #[tokio::test]
17792 async fn verification_is_standalone_inside_a_live_worker() {
17793 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17794 if std::env::var_os(CHILD).is_none() {
17795 let home = tempfile::tempdir().unwrap();
17796 let generation = crate::worker::pin_current(home.path()).unwrap();
17797 let output = std::process::Command::new(std::env::current_exe().unwrap())
17798 .args([
17799 "--exact",
17800 "graph::tests::verification_is_standalone_inside_a_live_worker",
17801 "--nocapture",
17802 ])
17803 .env(CHILD, "1")
17804 .env("MAGI_HOME", home.path())
17805 .env("MAGI_WORKER_HOME", home.path())
17806 .env("MAGI_WORKER_ID", "abcdef")
17807 .env("MAGI_WORKER_GENERATION", &generation.id)
17808 .env("MAGI_WORKER_EXE", &generation.executable)
17809 .env("MAGI_RUN", "verification-parent")
17810 .env("MAGI_NODE", "gate")
17811 .env("MAGI_SEAT", "worker-fixture")
17812 .env("MAGI_PROCESS_OWNER", "fixture-owner")
17813 .env("MAGI_NO_AUTOUPDATE", "0")
17814 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17815 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17816 .env(
17817 "VERIFY_EXPECT_TOOLCHAIN",
17818 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17819 )
17820 .env(
17821 "VERIFY_EXPECT_CACHE",
17822 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17823 )
17824 .output()
17825 .unwrap();
17826 assert!(
17827 output.status.success()
17828 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17829 "{}\n{}",
17830 String::from_utf8_lossy(&output.stdout),
17831 String::from_utf8_lossy(&output.stderr)
17832 );
17833 return;
17834 }
17835 let home = crate::run::home();
17836 let directory = home.join("workers/jobs/abcdef");
17837 std::fs::create_dir_all(&directory).unwrap();
17838 let job = crate::worker::Job {
17839 id: "abcdef".into(),
17840 sequence: 1,
17841 kind: "task".into(),
17842 key: "task:verification".into(),
17843 payload: serde_json::json!({}),
17844 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17845 version: Some(env!("CARGO_PKG_VERSION").into()),
17846 status: crate::worker::Status::Running,
17847 result: None,
17848 pid: Some(std::process::id()),
17849 created_at: 0,
17850 finished_at: None,
17851 };
17852 std::fs::write(
17853 directory.join("job.json"),
17854 serde_json::to_vec(&job).unwrap(),
17855 )
17856 .unwrap();
17857 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17858 let assert_worker = || {
17859 assert_eq!(
17860 crate::worker::current_job(&home).unwrap().unwrap().id,
17861 job.id
17862 );
17863 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17864 assert_eq!(
17865 crate::worker::helper_executable(&home).unwrap(),
17866 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17867 );
17868 assert!(
17869 crate::updater::Checker::new(&crate::config::Update {
17870 mode: crate::config::UpdateMode::Notify,
17871 interval: None,
17872 auto_restart: false,
17873 })
17874 .is_none(),
17875 "the driver remains pinned, even while tests are standalone"
17876 );
17877 };
17878 assert_worker();
17879 let repo = home.join("repo");
17880 std::fs::create_dir_all(&repo).unwrap();
17881 init_repo(&repo);
17882 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17885 test -z "${MAGI_WORKER_GENERATION+x}" &&
17886 test -z "${MAGI_WORKER_HOME+x}" &&
17887 test -z "${MAGI_WORKER_EXE+x}" &&
17888 test "$MAGI_HOME/repo" -ef . &&
17889 test "$MAGI_RUN" = verification-parent &&
17890 test "$MAGI_NODE" = gate &&
17891 test "$MAGI_SEAT" = worker-fixture &&
17892 test -n "$MAGI_PROCESS_OWNER" &&
17893 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17894 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17895 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17896 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17897 assert!(
17899 !std::process::Command::new("sh")
17900 .args(["-c", probe])
17901 .current_dir(&repo)
17902 .status()
17903 .unwrap()
17904 .success()
17905 );
17906 let unrelated_home = tempfile::tempdir().unwrap();
17907 assert!(
17908 crate::worker::current_job(unrelated_home.path()).is_err(),
17909 "an inherited worker id cannot be resolved in a unit fixture's home"
17910 );
17911 let mut runner = runner_at(RunStatus::Gating);
17912 runner.state.repo = repo.clone();
17913 runner.state.config.disk.min_free_bytes = 0;
17914 runner.state.config.coordination = crate::config::test_coordination();
17915 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17916 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17917 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17918 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17919 let winner = Candidate {
17920 index: 0,
17921 label: 'A',
17922 agent: "fixture".into(),
17923 branch: "main".into(),
17924 worktree: repo.clone(),
17925 summary: String::new(),
17926 stat: String::new(),
17927 files: 0,
17928 commits: 0,
17929 empty: false,
17930 failed: None,
17931 verified_noop: None,
17932 duration_ms: 0,
17933 folded: false,
17934 };
17935 let (e2e, retried, pids) = run_e2e_with_retry(
17936 &mut runner.state,
17937 &["sh".into(), "-c".into()],
17938 &[format!("{probe} && printf initial")],
17939 &repo,
17940 Duration::from_secs(10),
17941 "initial",
17942 )
17943 .await;
17944 assert!(!retried && pids.is_empty());
17945 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17946 assert_eq!(e2e[0].output_tail, "initial");
17947 let gate = runner.run_gate(&winner).await.unwrap();
17948 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17949 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17952 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17953 passed: false,
17954 outcomes: Vec::new(),
17955 });
17956 runner.state.save().unwrap();
17957 let mut resumed = RunState::load(&runner.state.id).unwrap();
17958 let wait = crate::repo_lock::WaitPolicy::default();
17959 assert!(matches!(
17960 check_gate_repair_e2e(&mut resumed, &winner, &wait)
17961 .await
17962 .unwrap(),
17963 GateFix::Retry
17964 ));
17965 let saved = RunState::load(&resumed.id).unwrap();
17966 let repair = saved.gate_repair_e2e.unwrap();
17967 assert!(repair.passed);
17968 assert_eq!(repair.outcomes.len(), 1);
17969 assert!(repair.outcomes[0].ok());
17970 let landing = run_landing_gate(&mut resumed, &winner, &wait)
17971 .await
17972 .unwrap();
17973 assert_eq!(landing.len(), 1);
17974 assert!(landing[0].ok(), "{landing:?}");
17975 assert_eq!(
17976 std::fs::read_to_string(repo.join("phases")).unwrap(),
17977 "pre\ngate\ne2e\npre\ngate\n"
17978 );
17979 assert_worker();
17980 }
17981
17982 #[cfg(unix)]
17983 #[tokio::test]
17984 async fn verification_keeps_the_panic_and_the_full_output() {
17985 crate::run::pin_test_home();
17986 let tmp = tempfile::tempdir().unwrap();
17987 let mut config = Config::default();
17988 config.disk.min_free_bytes = 0;
17989 let mut state = RunState::new(
17990 tmp.path().to_owned(),
17991 "main".into(),
17992 "base".into(),
17993 "verification evidence".into(),
17994 config,
17995 );
17996 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
17997 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
17998 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
17999 let mut names = Vec::new();
18000 for _ in 0..2 {
18001 let (outcomes, _) = run_commands(
18002 &mut state,
18003 "gate",
18004 "gate",
18005 0,
18006 &["sh".into(), "-c".into()],
18007 &[script.into()],
18008 tmp.path(),
18009 Duration::from_secs(60),
18010 )
18011 .await;
18012 let tail = &outcomes[0].output_tail;
18013 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18014 assert!(tail.contains("boom assertion"));
18015 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18016 let line = tail.lines().next().unwrap();
18017 names.push(line.to_owned());
18018 }
18019 assert_ne!(names[0], names[1], "the second run must not overwrite");
18020 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18021 assert!(full.len() > 500_000 && full.contains("boom assertion"));
18022 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18023
18024 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18026 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18027 let (outcomes, _) = run_commands(
18028 &mut state,
18029 "gate",
18030 "gate",
18031 0,
18032 &["sh".into(), "-c".into()],
18033 &["echo failure-output; exit 7".into()],
18034 tmp.path(),
18035 Duration::from_secs(10),
18036 )
18037 .await;
18038 assert_eq!(outcomes[0].code, Some(7));
18039 assert!(state.events.iter().any(|e| {
18040 e.node == "gate" && e.message.contains("could not save the full gate output:")
18041 }));
18042 }
18043
18044 #[cfg(unix)]
18045 #[tokio::test]
18046 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18047 crate::run::pin_test_home();
18048 let tmp = tempfile::tempdir().unwrap();
18049 let mut config = Config::default();
18050 config.disk.min_free_bytes = 0;
18051 let mut state = RunState::new(
18052 tmp.path().to_owned(),
18053 "main".into(),
18054 "base".into(),
18055 "verification evidence".into(),
18056 config,
18057 );
18058 let (outcomes, pids) = run_commands(
18059 &mut state,
18060 "gate",
18061 "gate",
18062 1,
18063 &["sh".into(), "-c".into()],
18064 &[
18065 "printf before-timeout; exec sleep 30".into(),
18066 "kill -TERM $$".into(),
18067 "exit 7".into(),
18068 "printf success".into(),
18069 ],
18070 tmp.path(),
18071 Duration::from_secs(1),
18072 )
18073 .await;
18074 wait_for_timed_out_children_to_die(&pids).await;
18075 assert_eq!(
18076 pids.len(),
18077 1,
18078 "unleased timeout retains its death-check PID"
18079 );
18080 assert_eq!(outcomes.len(), 4);
18081 assert_eq!(outcomes[0].code, None);
18082 assert!(outcomes[0].output_tail.contains("before-timeout"));
18083 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18084 assert_eq!(outcomes[1].code, None);
18085 assert!(outcomes[1].output_tail.contains("command terminated:"));
18086 assert!(!outcomes[1].output_tail.contains("timed out"));
18087 assert_eq!(outcomes[2].code, Some(7));
18088 assert!(!outcomes[2].output_tail.contains("timed out"));
18089 assert_eq!(outcomes[3].code, Some(0));
18090 assert_eq!(outcomes[3].output_tail, "success");
18091 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18092 }
18093
18094 #[tokio::test]
18104 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18105 crate::run::pin_test_home();
18106
18107 let tmp = tempfile::tempdir().expect("tempdir");
18108 let repo = tmp.path().join("repo");
18109 std::fs::create_dir_all(&repo).unwrap();
18110 init_repo(&repo);
18111
18112 let mut config = Config::default();
18113 config.verify.gate = vec![
18114 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18115 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18116 .to_owned(),
18117 ];
18118
18119 let mut state = RunState::new(
18120 repo.clone(),
18121 "main".to_owned(),
18122 "deadbeef".to_owned(),
18123 "task".to_owned(),
18124 config,
18125 );
18126 let run_id = state.id.clone();
18127 state.candidates = vec![Candidate {
18128 index: 0,
18129 label: 'A',
18130 agent: "alpha".to_owned(),
18131 branch: "does-not-exist".to_owned(),
18132 worktree: repo.clone(),
18133 summary: String::new(),
18134 stat: String::new(),
18135 files: 0,
18136 commits: 0,
18137 empty: false,
18138 failed: None,
18139 verified_noop: None,
18140 duration_ms: 0,
18141 folded: false,
18142 }];
18143 state.tally = Some(Tally {
18144 first_choice: BTreeMap::from([('A', 1)]),
18145 borda: BTreeMap::new(),
18146 winner: 'A',
18147 rankings: 1,
18148 unanimous_initial: true,
18149 deliberated: false,
18150 changed_votes: 0,
18151 unanimous_final: true,
18152 tie_break: None,
18153 judges: 0,
18154 present: 0,
18155 quorum: 0,
18156 met_quorum: true,
18157 uncontested: Some("only candidate A produced a change".to_owned()),
18158 });
18159 state.reviews = vec![ReviewRound {
18160 round: 1,
18161 head: "deadbeef".to_owned(),
18162 verified_head: None,
18163 verified_at: None,
18164 reviews: Vec::new(),
18165 e2e: Vec::new(),
18166 fix: None,
18167 blocking: 0,
18168 answered: 0,
18169 expected: 0,
18170 clean: true,
18171 verify_retried: false,
18172 e2e_deferred: false,
18173 e2e_defer_reason: None,
18174 progressed: false,
18175 vote_split: false,
18176 reconsideration: Vec::new(),
18177 verdict: None,
18178 }];
18179
18180 let mut runner = Runner {
18181 state,
18182 roles: ResolvedRoles {
18183 implementers: Vec::new(),
18184 judges: Vec::new(),
18185 reviewers: Vec::new(),
18186 fixer: None,
18187 conductor: conductor(),
18188 implementer_roster: Vec::new(),
18189 judge_roster: Vec::new(),
18190 reviewer_roster: Vec::new(),
18191 },
18192 sem: Arc::new(Semaphore::new(1)),
18193 pause: Pause::new(),
18194 interrupt: Pause::new(),
18195 };
18196
18197 let started_marker = repo.join("started.marker");
18198 let release_marker = repo.join("release.marker");
18199 let poller = tokio::spawn(async move {
18200 for _ in 0..100 {
18205 if started_marker.exists()
18206 && let Ok(s) = crate::run::RunState::load(&run_id)
18207 && let Some(a) = s.active.get("gate")
18208 {
18209 std::fs::write(&release_marker, b"go").expect("release marker");
18210 return Some(a.clone());
18211 }
18212 tokio::time::sleep(Duration::from_millis(50)).await;
18213 }
18214 None
18215 });
18216
18217 runner.gate().await.expect("gate");
18218 let captured = poller.await.expect("poller task");
18219 let captured = captured.expect(
18220 "the poller never saw a `gate` task entry in run.json while the command was \
18221 still blocked on its own release marker",
18222 );
18223
18224 assert_eq!(captured.task.as_deref(), Some("gate"));
18225 assert_eq!(captured.node, "gate");
18226 assert_eq!(captured.index, Some(1));
18227 assert_eq!(captured.total, Some(1));
18228 assert!(
18229 captured
18230 .command
18231 .as_deref()
18232 .is_some_and(|c| c.contains("started.marker")),
18233 "{captured:?}"
18234 );
18235
18236 assert!(
18237 runner.state.active.is_empty(),
18238 "the entry must be cleared once the command actually finished: {:?}",
18239 runner.state.active
18240 );
18241 assert!(runner.state.gate_ran);
18242 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18243 }
18244
18245 #[tokio::test]
18248 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18249 use crate::verdict::{Finding, ReviewVote, Severity};
18250 crate::run::pin_test_home();
18251 let tmp = tempfile::tempdir().expect("tempdir");
18252 let repo = tmp.path().join("repo");
18253 std::fs::create_dir_all(&repo).unwrap();
18254 init_repo(&repo);
18255
18256 for (severity, vote, expect) in [
18257 (Severity::Major, ReviewVote::Reject, true),
18258 (Severity::Minor, ReviewVote::Reject, false),
18259 (Severity::Major, ReviewVote::Approve, false),
18260 ] {
18261 let mut round = review_round(false, 1, 1, 1, false, true);
18262 round.reviews = vec![ReviewRecord {
18263 reviewer: 1,
18264 agent: "alpha".to_owned(),
18265 summary: String::new(),
18266 findings: vec![Finding {
18267 id: "R1-1-1".to_owned(),
18268 severity,
18269 file: None,
18270 line: None,
18271 title: "t".to_owned(),
18272 detail: String::new(),
18273 }],
18274 vote: Some(vote),
18275 failed: None,
18276 duration_ms: 0,
18277 attempts: 0,
18278 }];
18279 let mut state = RunState::new(
18280 repo.clone(),
18281 "main".to_owned(),
18282 "deadbeef".to_owned(),
18283 "task".to_owned(),
18284 Config::default(),
18285 );
18286 state.reviews = vec![round];
18287 let mut runner = Runner {
18288 state,
18289 roles: ResolvedRoles {
18290 implementers: Vec::new(),
18291 judges: Vec::new(),
18292 reviewers: Vec::new(),
18293 fixer: None,
18294 conductor: conductor(),
18295 implementer_roster: Vec::new(),
18296 judge_roster: Vec::new(),
18297 reviewer_roster: Vec::new(),
18298 },
18299 sem: Arc::new(Semaphore::new(1)),
18300 pause: Pause::new(),
18301 interrupt: Pause::new(),
18302 };
18303 let shell = runner.state.config.shell();
18304 runner
18305 .stop_reviewing("round budget spent", &shell, &repo)
18306 .await
18307 .expect("stop_reviewing");
18308 assert_eq!(runner.state.status, RunStatus::Gating);
18309 assert_eq!(
18310 runner.state.contested_handoff.is_some(),
18311 expect,
18312 "{severity:?} + {vote:?}"
18313 );
18314 }
18315 }
18316
18317 #[tokio::test]
18330 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18331 crate::run::pin_test_home();
18332 let home = crate::run::home();
18333
18334 let tmp = tempfile::tempdir().expect("tempdir");
18335 let repo = tmp.path().join("repo");
18336 std::fs::create_dir_all(&repo).unwrap();
18337 init_repo(&repo);
18338 let head = crate::git::rev_parse(&repo, "HEAD")
18339 .await
18340 .expect("rev-parse");
18341 let cache_dir = tmp.path().join("target");
18344
18345 let mut config = Config::default();
18346 config.verify.e2e = vec![format!(
18347 "CARGO_TARGET_DIR='{}' test -f README.md",
18348 cache_dir.display()
18349 )];
18350 config.graph.review_rounds = 1;
18351 config.graph.timeout_verify = Some(2);
18354 config.graph.timeout_resource_wait = 2;
18355
18356 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18357 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18358 .expect("no io error acquiring directly")
18359 {
18360 crate::cache::AcquireOutcome::Acquired(g) => g,
18361 crate::cache::AcquireOutcome::Busy(b) => {
18362 panic!("expected the direct acquire to win the lease first: {b:?}")
18363 }
18364 };
18365
18366 let mut state = RunState::new(
18367 repo.clone(),
18368 "main".to_owned(),
18369 head.clone(),
18370 "task".to_owned(),
18371 config,
18372 );
18373 state.candidates = vec![Candidate {
18374 index: 0,
18375 label: 'A',
18376 agent: "alpha".to_owned(),
18377 branch: "does-not-exist".to_owned(),
18378 worktree: repo.clone(),
18379 summary: String::new(),
18380 stat: String::new(),
18381 files: 0,
18382 commits: 0,
18383 empty: false,
18384 failed: None,
18385 verified_noop: None,
18386 duration_ms: 0,
18387 folded: false,
18388 }];
18389 state.tally = Some(Tally {
18390 first_choice: BTreeMap::from([('A', 1)]),
18391 borda: BTreeMap::new(),
18392 winner: 'A',
18393 rankings: 1,
18394 unanimous_initial: true,
18395 deliberated: false,
18396 changed_votes: 0,
18397 unanimous_final: true,
18398 tie_break: None,
18399 judges: 0,
18400 present: 0,
18401 quorum: 0,
18402 met_quorum: true,
18403 uncontested: Some("only candidate A produced a change".to_owned()),
18404 });
18405 state.reviews = vec![ReviewRound {
18409 round: 1,
18410 head: head.clone(),
18411 verified_head: None,
18412 verified_at: None,
18413 reviews: Vec::new(),
18414 e2e: Vec::new(),
18415 fix: None,
18416 blocking: 1,
18417 answered: 1,
18418 expected: 1,
18419 clean: false,
18420 verify_retried: false,
18421 e2e_deferred: true,
18422 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18423 progressed: false,
18424 vote_split: false,
18425 reconsideration: Vec::new(),
18426 verdict: None,
18427 }];
18428
18429 let mut runner = Runner {
18430 state,
18431 roles: ResolvedRoles {
18432 implementers: Vec::new(),
18433 judges: Vec::new(),
18434 reviewers: Vec::new(),
18435 fixer: None,
18436 conductor: conductor(),
18437 implementer_roster: Vec::new(),
18438 judge_roster: Vec::new(),
18439 reviewer_roster: Vec::new(),
18440 },
18441 sem: Arc::new(Semaphore::new(1)),
18442 pause: Pause::new(),
18443 interrupt: Pause::new(),
18444 };
18445
18446 let shell = runner.state.config.shell();
18447 runner
18448 .stop_reviewing("round budget spent", &shell, &repo)
18449 .await
18450 .expect("stop_reviewing");
18451
18452 let last = runner.state.reviews.last().expect("round record");
18453 assert_eq!(
18454 last.e2e_status(),
18455 E2eStatus::ResourceBlocked,
18456 "the shared cache is still held; the attempt must read as blocked, not deferred or \
18457 failed: {last:?}"
18458 );
18459 assert_eq!(
18460 last.verified_head.as_deref(),
18461 Some(head.as_str()),
18462 "which commit this attempt targeted is known even though nothing finished checking \
18463 it"
18464 );
18465 let first_attempt_at = last
18466 .verified_at
18467 .expect("when this attempt ran is known too");
18468 assert_ne!(
18469 runner.state.status,
18470 RunStatus::Blocked,
18471 "contention is evidence about the machine, not the patch — it must not settle the \
18472 run as blocked: {:?}",
18473 runner.state.status
18474 );
18475 assert!(
18476 !runner
18477 .state
18478 .events
18479 .iter()
18480 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18481 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18482 runner.state.events
18483 );
18484
18485 runner
18490 .stop_reviewing("round budget spent", &shell, &repo)
18491 .await
18492 .expect("stop_reviewing retry");
18493 assert_eq!(
18494 runner.state.reviews.len(),
18495 1,
18496 "no new round was started: {:?}",
18497 runner.state.reviews
18498 );
18499 let last = runner.state.reviews.last().expect("round record");
18500 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18501 assert!(
18502 last.verified_at.expect("still known") > first_attempt_at,
18503 "a second reentry must be a fresh attempt, not a stale copy of the first"
18504 );
18505 assert_ne!(runner.state.status, RunStatus::Blocked);
18506
18507 held.release();
18508 }
18509
18510 #[cfg(unix)]
18511 #[tokio::test]
18512 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
18513 ask_test_home();
18514 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
18515 let tmp = tempfile::tempdir().unwrap();
18516 let repo = tmp.path().join("repo");
18517 std::fs::create_dir(&repo).unwrap();
18518 init_repo(&repo);
18519 let run_git = |args: &[&str]| {
18520 let out = std::process::Command::new("git")
18521 .args(args)
18522 .current_dir(&repo)
18523 .output()
18524 .unwrap();
18525 assert!(
18526 out.status.success(),
18527 "{args:?}: {}",
18528 String::from_utf8_lossy(&out.stderr)
18529 );
18530 };
18531 let origin = tmp.path().join("origin.git");
18532 run_git(&["init", "--bare", origin.to_str().unwrap()]);
18533 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
18534 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
18535 run_git(&["add", "spec.rs"]);
18536 run_git(&["commit", "-m", "initial spec"]);
18537 run_git(&["push", "origin", "main"]);
18538 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
18539 run_git(&["checkout", "-b", "fix/review"]);
18540 std::fs::write(
18541 repo.join("check.rs"),
18542 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
18543 )
18544 .unwrap();
18545 run_git(&["add", "check.rs"]);
18546 run_git(&["commit", "-m", "use spec"]);
18547 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
18548 run_git(&["checkout", "main"]);
18549 std::fs::write(
18550 repo.join("spec.rs"),
18551 "struct AgentSpec { max_turn_secs: u64 }\n",
18552 )
18553 .unwrap();
18554 run_git(&["add", "spec.rs"]);
18555 run_git(&["commit", "-m", "extend spec"]);
18556 run_git(&["push", "origin", "main"]);
18557 run_git(&["checkout", "fix/review"]);
18558
18559 let mut runner = runner_at(RunStatus::Reviewing);
18560 runner.state.repo = repo.clone();
18561 runner.state.base_commit = base;
18562 runner.state.pr_head = Some("fix/review".into());
18563 runner.state.config.graph.review_rounds = max_rounds;
18564 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
18565 runner.state.config.graph.sessions = false;
18566 runner.state.config.disk.min_free_bytes = 0;
18567 runner.state.config.verify.e2e =
18568 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
18569 runner.state.candidates = vec![Candidate {
18570 index: 0,
18571 label: 'A',
18572 agent: "fixture".into(),
18573 branch: "fix/review".into(),
18574 worktree: repo.clone(),
18575 summary: String::new(),
18576 stat: String::new(),
18577 files: 1,
18578 commits: 1,
18579 empty: false,
18580 failed: None,
18581 verified_noop: None,
18582 duration_ms: 0,
18583 folded: false,
18584 }];
18585 runner.state.tally = Some(Tally {
18586 first_choice: BTreeMap::new(),
18587 borda: BTreeMap::new(),
18588 winner: 'A',
18589 rankings: 0,
18590 unanimous_initial: true,
18591 deliberated: false,
18592 changed_votes: 0,
18593 unanimous_final: true,
18594 tie_break: None,
18595 judges: 0,
18596 present: 0,
18597 quorum: 0,
18598 met_quorum: true,
18599 uncontested: Some("review-only".into()),
18600 });
18601 let mut last = review_round(false, 0, 1, 1, false, false);
18602 last.head = reviewed.clone();
18603 last.e2e_deferred = true;
18604 last.e2e[0].resource_blocked = true;
18605 last.e2e[0].output_tail = "waiting for the shared build cache".into();
18606 if max_rounds == 1 {
18607 last.e2e.clear();
18608 }
18609 runner.state.reviews = vec![last];
18610 let calls = tmp.path().join("fix-calls");
18611 let mut fixer = spec("repair");
18612 fixer
18613 .env
18614 .insert("FIX_CALLS".into(), calls.display().to_string());
18615 fixer.command = vec!["sh".into(), "-c".into(), r#"
18616 echo fix >> "$FIX_CALLS"
18617 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18618 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18619 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18620 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18621 "#.into()];
18622 let mut reviewer = spec("reviewer");
18623 reviewer.command = vec![
18624 "sh".into(),
18625 "-c".into(),
18626 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18627 .into(),
18628 ];
18629 runner.roles.fixer = Some(vec![fixer]);
18630 runner.roles.reviewers = vec![reviewer];
18631
18632 runner.sync_to_base().await.unwrap();
18633 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18634 assert_ne!(rebased, reviewed);
18635 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18636 if reentry {
18637 let shell = runner.state.config.shell();
18640 runner
18641 .stop_unresolved_review("deferred verification", &shell, &repo)
18642 .await
18643 .unwrap();
18644 assert!(runner.catchup_repairable());
18645 assert!(!calls.exists());
18646 runner.state = RunState::load(&runner.state.id).unwrap();
18647 assert!(runner.catchup_repairable());
18648 }
18649 runner.review_loop().await.unwrap();
18650 let first = &runner.state.reviews[0];
18651 assert_eq!(first.head, reviewed);
18652 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18653 assert!(
18654 first.verify_retried,
18655 "a reproduced build failure gets exactly one retry: {:?}",
18656 first.e2e
18657 );
18658 assert!(first.e2e[0].output_tail.contains("E0063"));
18659 assert!(!runner.catchup_repairable());
18660 if max_rounds == 2 {
18661 assert_eq!(
18662 runner.state.status,
18663 RunStatus::Gating,
18664 "{}",
18665 crate::report::run(&runner.state)
18666 );
18667 assert_eq!(runner.state.reviews.len(), 2);
18668 assert!(first.fix.as_ref().unwrap().committed);
18669 assert!(runner.state.reviews[1].clean);
18670 assert_eq!(
18671 runner
18672 .state
18673 .reviews
18674 .iter()
18675 .filter(|r| r.fix.is_some())
18676 .count(),
18677 1
18678 );
18679 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18680 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18681 } else {
18682 assert_eq!(runner.state.status, RunStatus::Blocked);
18683 assert!(first.fix.is_none());
18684 assert!(!calls.exists());
18685 let recovery = crate::recovery::assess(
18686 "prior",
18687 Some(&runner.state),
18688 &runner.landing_base(),
18689 crate::recovery::Relation::Diverged,
18690 );
18691 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18692 let reasons = recovery.reasons.join("\n");
18693 assert!(
18694 reasons.contains(&reviewed) && reasons.contains(&rebased),
18695 "{reasons}"
18696 );
18697 assert!(
18698 reasons.contains("base sync to")
18699 && reasons.contains("additional review is required"),
18700 "{reasons}"
18701 );
18702 }
18703 }
18704 }
18705
18706 #[tokio::test]
18718 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18719 crate::run::pin_test_home();
18720 let home = crate::run::home();
18721
18722 let tmp = tempfile::tempdir().expect("tempdir");
18723 let repo = tmp.path().join("repo");
18724 std::fs::create_dir_all(&repo).unwrap();
18725 init_repo(&repo);
18726 let head = crate::git::rev_parse(&repo, "HEAD")
18727 .await
18728 .expect("rev-parse");
18729 let cache_dir = tmp.path().join("target");
18730
18731 let mut config = Config::default();
18732 config.verify.e2e = vec![format!(
18733 "CARGO_TARGET_DIR='{}' test -f README.md",
18734 cache_dir.display()
18735 )];
18736 config.graph.review_rounds = 1;
18737 config.graph.timeout_verify = Some(2);
18738 config.graph.timeout_resource_wait = 2;
18739
18740 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18741 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18742 .expect("no io error acquiring directly")
18743 {
18744 crate::cache::AcquireOutcome::Acquired(g) => g,
18745 crate::cache::AcquireOutcome::Busy(b) => {
18746 panic!("expected the direct acquire to win the lease first: {b:?}")
18747 }
18748 };
18749
18750 let mut state = RunState::new(
18751 repo.clone(),
18752 "main".to_owned(),
18753 head.clone(),
18754 "task".to_owned(),
18755 config,
18756 );
18757 state.candidates = vec![Candidate {
18758 index: 0,
18759 label: 'A',
18760 agent: "alpha".to_owned(),
18761 branch: "does-not-exist".to_owned(),
18762 worktree: repo.clone(),
18763 summary: String::new(),
18764 stat: String::new(),
18765 files: 0,
18766 commits: 0,
18767 empty: false,
18768 failed: None,
18769 verified_noop: None,
18770 duration_ms: 0,
18771 folded: false,
18772 }];
18773 state.tally = Some(Tally {
18774 first_choice: BTreeMap::from([('A', 1)]),
18775 borda: BTreeMap::new(),
18776 winner: 'A',
18777 rankings: 1,
18778 unanimous_initial: true,
18779 deliberated: false,
18780 changed_votes: 0,
18781 unanimous_final: true,
18782 tie_break: None,
18783 judges: 0,
18784 present: 0,
18785 quorum: 0,
18786 met_quorum: true,
18787 uncontested: Some("only candidate A produced a change".to_owned()),
18788 });
18789 state.reviews = vec![ReviewRound {
18793 round: 1,
18794 head: head.clone(),
18795 verified_head: Some(head.clone()),
18796 verified_at: Some(jiff::Timestamp::now()),
18797 reviews: Vec::new(),
18798 e2e: vec![CommandOutcome {
18799 command: format!(
18800 "CARGO_TARGET_DIR='{}' test -f README.md",
18801 cache_dir.display()
18802 ),
18803 code: None,
18804 output_tail: "waiting for the shared build cache".to_owned(),
18805 duration_ms: 0,
18806 resource_blocked: true,
18807 }],
18808 fix: None,
18809 blocking: 1,
18810 answered: 1,
18811 expected: 1,
18812 clean: false,
18813 verify_retried: false,
18814 e2e_deferred: false,
18815 e2e_defer_reason: None,
18816 progressed: false,
18817 vote_split: false,
18818 reconsideration: Vec::new(),
18819 verdict: None,
18820 }];
18821
18822 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18823 let mut runner = Runner {
18824 state,
18825 roles: ResolvedRoles {
18826 implementers: Vec::new(),
18827 judges: Vec::new(),
18828 reviewers: Vec::new(),
18829 fixer: None,
18830 conductor: conductor(),
18831 implementer_roster: Vec::new(),
18832 judge_roster: Vec::new(),
18833 reviewer_roster: Vec::new(),
18834 },
18835 sem: Arc::new(Semaphore::new(1)),
18836 pause: Pause::new(),
18837 interrupt: Pause::new(),
18838 };
18839
18840 runner.review_loop().await.expect("review_loop");
18845
18846 assert_eq!(
18847 runner.state.reviews.len(),
18848 1,
18849 "no new round was started on top of the unresolved one: {:?}",
18850 runner.state.reviews
18851 );
18852 let last = &runner.state.reviews[0];
18853 assert_eq!(
18854 last.e2e_status(),
18855 E2eStatus::ResourceBlocked,
18856 "still contended: {last:?}"
18857 );
18858 assert!(
18859 last.verified_at.expect("still known") > first_attempt_at,
18860 "review_loop must have actually retried the check, not left it exactly as found"
18861 );
18862 assert_ne!(
18863 runner.state.status,
18864 RunStatus::Blocked,
18865 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18866 runner.state.status
18867 );
18868
18869 held.release();
18870 }
18871
18872 #[tokio::test]
18873 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18874 crate::run::pin_test_home();
18875 for status in [RunStatus::Blocked, RunStatus::Gating] {
18876 let mut runner = runner_at(status);
18877 runner.state.config.graph.land = true;
18878 runner.state.gate_ran = true;
18879 runner.state.merge = Some(MergeOutcome {
18880 mode: MergeMode::Pr,
18881 ok: false,
18882 detail: "the fixer produced no commit".into(),
18883 empty: false,
18884 });
18885 let url = "https://example.invalid/o/r/pull/1";
18886 runner.state.pr = Some(crate::run::PrRecord {
18887 url: url.into(),
18888 number: 1,
18889 state: "open".into(),
18890 checks: "red".into(),
18891 round: 1,
18892 rounds: 1,
18893 red_at_merge: Vec::new(),
18894 });
18895 runner.state.save().unwrap();
18896 runner.execute_graph().await.unwrap();
18900 assert_eq!(runner.state.status, RunStatus::Blocked);
18901 assert!(runner.state.land_sync_pending);
18902 assert!(runner.state.events.iter().any(|event|
18903 event.node == "land" && event.message == format!("watching {url}")));
18904 assert_eq!(
18905 runner.state.merge.as_ref().unwrap().detail,
18906 "the fixer produced no commit"
18907 );
18908 }
18909 }
18910
18911 #[tokio::test]
18912 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18913 crate::run::pin_test_home();
18914 let tmp = tempfile::tempdir().expect("tempdir");
18915 let repo = tmp.path().join("repo");
18916 std::fs::create_dir_all(&repo).unwrap();
18917 init_repo(&repo);
18918
18919 let mut config = Config::default();
18920 config.merge.mode = MergeMode::Pr;
18921 config.graph.land = true;
18922 config.graph.land_approval = false;
18923
18924 let mut state = RunState::new(
18925 repo.clone(),
18926 "main".to_owned(),
18927 "deadbeef".to_owned(),
18928 "task".to_owned(),
18929 config,
18930 );
18931 state.candidates = vec![Candidate {
18932 index: 0,
18933 label: 'A',
18934 agent: "alpha".to_owned(),
18935 branch: "does-not-exist".to_owned(),
18936 worktree: repo.clone(),
18937 summary: String::new(),
18938 stat: String::new(),
18939 files: 0,
18940 commits: 0,
18941 empty: false,
18942 failed: None,
18943 verified_noop: None,
18944 duration_ms: 0,
18945 folded: false,
18946 }];
18947 state.tally = Some(Tally {
18948 first_choice: BTreeMap::from([('A', 1)]),
18949 borda: BTreeMap::new(),
18950 winner: 'A',
18951 rankings: 1,
18952 unanimous_initial: true,
18953 deliberated: false,
18954 changed_votes: 0,
18955 unanimous_final: true,
18956 tie_break: None,
18957 judges: 0,
18958 present: 0,
18959 quorum: 0,
18960 met_quorum: true,
18961 uncontested: Some("only candidate A produced a change".to_owned()),
18962 });
18963 state.reviews = vec![ReviewRound {
18964 round: 1,
18965 head: "deadbeef".to_owned(),
18966 verified_head: None,
18967 verified_at: None,
18968 reviews: Vec::new(),
18969 e2e: Vec::new(),
18970 fix: None,
18971 blocking: 0,
18972 answered: 0,
18973 expected: 0,
18974 clean: true,
18975 verify_retried: false,
18976 e2e_deferred: false,
18977 e2e_defer_reason: None,
18978 progressed: false,
18979 vote_split: false,
18980 reconsideration: Vec::new(),
18981 verdict: None,
18982 }];
18983 state.gate = vec![CommandOutcome {
18984 command: "test".to_owned(),
18985 code: Some(0),
18986 output_tail: String::new(),
18987 duration_ms: 0,
18988 resource_blocked: false,
18989 }];
18990 state.gate_ran = true;
18991 state.status = RunStatus::Landing;
18995 state.merge = Some(MergeOutcome {
18996 mode: MergeMode::Pr,
18997 ok: true,
18998 detail: "https://example.invalid/x/y/pull/1".to_owned(),
18999 empty: false,
19000 });
19001
19002 ask_test_home();
19006 state.save().expect("save landing run");
19009 let store = ask::Questions::open();
19010 let q = ask_open_question(&store, &state.id);
19011
19012 let mut runner = Runner {
19013 state,
19014 roles: ResolvedRoles {
19015 implementers: Vec::new(),
19016 judges: Vec::new(),
19017 reviewers: Vec::new(),
19018 fixer: None,
19019 conductor: conductor(),
19020 implementer_roster: Vec::new(),
19021 judge_roster: Vec::new(),
19022 reviewer_roster: Vec::new(),
19023 },
19024 sem: Arc::new(Semaphore::new(1)),
19025 pause: Pause::new(),
19026 interrupt: Pause::new(),
19027 };
19028
19029 runner.execute().await.expect("execute");
19034
19035 assert_eq!(
19036 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19037 Some("https://example.invalid/x/y/pull/1"),
19038 "reentry must not push again or open a second pull request over the \
19039 one `land` is already watching"
19040 );
19041 assert_ne!(
19042 runner.state.status,
19043 RunStatus::Landing,
19044 "land could not actually reach the fake pull request, so it must \
19045 have given up rather than left the run silently parked forever"
19046 );
19047 assert_eq!(runner.state.status, RunStatus::Blocked);
19051 assert!(
19052 store.get(&q.id).unwrap().status.open(),
19053 "Blocked is still alive; settle_questions must have been a no-op here"
19054 );
19055 }
19056
19057 fn state_with_round(round: ReviewRound) -> RunState {
19058 let mut s = RunState::new(
19059 PathBuf::from("/repo"),
19060 "main".to_owned(),
19061 "abc1234".to_owned(),
19062 "add retries".to_owned(),
19063 Config::default(),
19064 );
19065 s.reviews = vec![round];
19066 s
19067 }
19068
19069 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19070 crate::verdict::Finding {
19071 id: id.to_owned(),
19072 severity,
19073 file: None,
19074 line: None,
19075 title: title.to_owned(),
19076 detail: String::new(),
19077 }
19078 }
19079
19080 #[test]
19081 fn pr_body_names_open_findings_and_declined_ones() {
19082 let round = ReviewRound {
19083 round: 2,
19084 head: "deadbee".to_owned(),
19085 verified_head: None,
19086 verified_at: None,
19087 reviews: vec![ReviewRecord {
19088 attempts: 0,
19089 reviewer: 1,
19090 agent: "alpha".to_owned(),
19091 summary: String::new(),
19092 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19093 vote: None,
19094 failed: None,
19095 duration_ms: 0,
19096 }],
19097 e2e: vec![CommandOutcome {
19098 command: "cargo test".to_owned(),
19099 code: Some(0),
19100 output_tail: String::new(),
19101 duration_ms: 0,
19102 resource_blocked: false,
19103 }],
19104 verify_retried: false,
19105 e2e_deferred: false,
19106 e2e_defer_reason: None,
19107 fix: Some(FixRecord {
19108 agent: "alpha".to_owned(),
19109 addressed: Vec::new(),
19110 rejected: vec![crate::verdict::Rejection {
19111 id: "R1-1-1".to_owned(),
19112 why: "not reachable from any caller".to_owned(),
19113 }],
19114 notes: String::new(),
19115 committed: true,
19116 failed: None,
19117 duration_ms: 0,
19118 continuation: None,
19119 }),
19120 blocking: 0,
19121 answered: 1,
19122 expected: 1,
19123 clean: false,
19124 progressed: true,
19125 vote_split: false,
19126 reconsideration: Vec::new(),
19127 verdict: None,
19128 };
19129 let state = state_with_round(round);
19130 let body = pr_message(&state, 'A').body;
19131
19132 assert!(body.contains("add retries"), "the task must still be there");
19133 assert!(body.contains("R2-1-1"), "{body}");
19134 assert!(!body.contains("unused import"), "{body}");
19135 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19136 assert!(
19137 !body.contains("not reachable from any caller"),
19138 "the reason it was declined: {body}"
19139 );
19140 }
19141
19142 #[test]
19143 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19144 let round = ReviewRound {
19145 round: 1,
19146 head: "deadbee".to_owned(),
19147 verified_head: None,
19148 verified_at: None,
19149 reviews: vec![ReviewRecord {
19150 attempts: 0,
19151 reviewer: 1,
19152 agent: "alpha".to_owned(),
19153 summary: String::new(),
19154 findings: Vec::new(),
19155 vote: None,
19156 failed: None,
19157 duration_ms: 0,
19158 }],
19159 e2e: Vec::new(),
19160 verify_retried: false,
19161 e2e_deferred: false,
19162 e2e_defer_reason: None,
19163 fix: None,
19164 blocking: 0,
19165 answered: 1,
19166 expected: 1,
19167 clean: true,
19168 progressed: false,
19169 vote_split: false,
19170 reconsideration: Vec::new(),
19171 verdict: None,
19172 };
19173 let state = state_with_round(round);
19174 let body = pr_message(&state, 'A').body;
19175 assert!(!body.contains("Open review findings"), "{body}");
19176 assert!(!body.contains("Declined"), "{body}");
19177 }
19178
19179 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19180 let mut state = RunState::new(
19181 PathBuf::from("/repo"),
19182 "main".to_owned(),
19183 "abc1234".to_owned(),
19184 instruction.to_owned(),
19185 Config::default(),
19186 );
19187 state.candidates.push(Candidate {
19188 index: 0,
19189 label: 'A',
19190 agent: "alpha".to_owned(),
19191 branch: "magi/x/A".to_owned(),
19192 worktree: PathBuf::from("/wt"),
19193 summary: summary.to_owned(),
19194 stat: String::new(),
19195 files: 1,
19196 commits: 1,
19197 empty: false,
19198 failed: None,
19199 verified_noop: None,
19200 folded: false,
19201 duration_ms: 0,
19202 });
19203 state
19204 }
19205
19206 fn review_state(subjects: &[&str]) -> RunState {
19207 let mut state = state_with_summary(
19208 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19209 "",
19210 );
19211 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19212 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19213 state
19214 }
19215
19216 #[tokio::test]
19219 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19220 ask_test_home();
19221 let tmp = tempfile::tempdir().unwrap();
19222 let repo = tmp.path().join("repo");
19223 std::fs::create_dir_all(&repo).unwrap();
19224 init_repo(&repo);
19225 let origin = tmp.path().join("origin.git");
19226 let g = |dir: &Path, args: &[&str]| {
19227 let out = std::process::Command::new("git")
19228 .args(args)
19229 .current_dir(dir)
19230 .quiet()
19231 .output()
19232 .expect("spawn git");
19233 assert!(
19234 out.status.success(),
19235 "git {args:?}: {}",
19236 String::from_utf8_lossy(&out.stderr)
19237 );
19238 };
19239 g(
19240 tmp.path(),
19241 &[
19242 "clone",
19243 "--bare",
19244 "-q",
19245 repo.to_str().unwrap(),
19246 origin.to_str().unwrap(),
19247 ],
19248 );
19249 g(
19250 &repo,
19251 &["remote", "add", "origin", origin.to_str().unwrap()],
19252 );
19253 let c1 = git::rev_parse(&repo, "main").await.unwrap();
19254
19255 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19257 g(&repo, &["add", "-A"]);
19258 g(
19259 &repo,
19260 &["commit", "-q", "-m", "chore(deps): update a crate"],
19261 );
19262 g(&repo, &["push", "-q", "origin", "main"]);
19263 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19264 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19265 g(&repo, &["add", "-A"]);
19266 g(
19267 &repo,
19268 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19269 );
19270 g(&repo, &["checkout", "-q", "main"]);
19271
19272 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19273 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19274 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19276 assert_eq!(fallback, c1);
19277
19278 let mut state = review_state(&[
19279 "chore(deps): update a crate",
19280 "fix(daemon): apply a chosen action",
19281 ]);
19282 state.repo = repo.clone();
19283 state.base_branch = "main".to_owned();
19284 state.base_commit = c1;
19285 refresh_reviewed_commits(&mut state, "feat/own").await;
19286 assert_eq!(
19287 state.reviewed_commits,
19288 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19289 );
19290 assert_eq!(
19291 review_title(&state).as_deref(),
19292 Some("fix(daemon): apply a chosen action")
19293 );
19294 assert_eq!(
19295 leaked_subjects(&state, "feat/own").await,
19296 Some(vec!["chore(deps): update a crate".to_owned()])
19297 );
19298 state.config.merge.remote = "nowhere".to_owned();
19301 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19302 }
19303
19304 struct SummaryFixture {
19308 _tmp: tempfile::TempDir,
19309 repo: PathBuf,
19310 reply: PathBuf,
19311 runner: Runner,
19312 }
19313
19314 fn summary_fixture() -> SummaryFixture {
19315 ask_test_home();
19316 let tmp = tempfile::tempdir().unwrap();
19317 let repo = tmp.path().join("repo");
19318 std::fs::create_dir_all(&repo).unwrap();
19319 init_repo(&repo);
19320 let origin = tmp.path().join("origin.git");
19321 let g = |args: &[&str]| {
19322 let out = std::process::Command::new("git")
19323 .args(args)
19324 .current_dir(&repo)
19325 .quiet()
19326 .output()
19327 .expect("spawn git");
19328 assert!(
19329 out.status.success(),
19330 "git {args:?}: {}",
19331 String::from_utf8_lossy(&out.stderr)
19332 );
19333 };
19334 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19335 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19336 g(&["push", "-q", "origin", "main"]);
19337 g(&["checkout", "-q", "-b", "feat/own"]);
19338 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19339 g(&["add", "-A"]);
19340 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19341 let reply = tmp.path().join("reply.txt");
19342 std::fs::write(
19343 &reply,
19344 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19345 )
19346 .unwrap();
19347 let mut runner = runner_at(RunStatus::Gating);
19348 let mut state = review_state(&["feat(own): add the own file"]);
19349 state.repo = repo.clone();
19350 state.base_branch = "main".to_owned();
19351 state.candidates[0].branch = "feat/own".to_owned();
19352 state.candidates[0].worktree = repo.clone();
19353 let mut author = spec("alpha");
19354 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19355 author
19356 .env
19357 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19358 state.config.agents = vec![author];
19359 runner.state = state;
19360 SummaryFixture {
19361 _tmp: tmp,
19362 repo,
19363 reply,
19364 runner,
19365 }
19366 }
19367
19368 async fn posted(f: &mut SummaryFixture) -> PrMessage {
19369 let winner = f.runner.state.candidates[0].clone();
19370 f.runner
19371 .guarded_pr_message(&winner, None, true)
19372 .await
19373 .unwrap()
19374 .expect("an English description never parks")
19375 }
19376
19377 fn clean_round_with_minor_findings() -> ReviewRound {
19378 let mut round = passing_round(2, None);
19379 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19380 round.reviews = vec![ReviewRecord {
19381 attempts: 0,
19382 reviewer: 1,
19383 agent: "alpha".into(),
19384 summary: "内部レビュー".into(),
19385 findings: vec![
19386 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19387 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19388 ],
19389 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19390 failed: None,
19391 duration_ms: 0,
19392 }];
19393 round
19394 }
19395
19396 #[tokio::test]
19397 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19398 for review_only in [false, true] {
19399 let mut f = summary_fixture();
19400 if !review_only {
19401 f.runner.state.candidates[0].agent = "alpha".into();
19402 }
19403 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19404 assert!(f.runner.state.reviews[0].clean);
19405 assert!(f.runner.state.reviews[0].fix.is_none());
19406 assert!(f.runner.state.open_findings().is_empty());
19407 assert_eq!(
19408 crate::pr_outcome::required_finding_ids(&f.runner.state),
19409 ["R2-1-1", "R2-1-2"]
19410 );
19411 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19412 let evidence = crate::pr_outcome::summary_evidence(
19413 &f.runner.state,
19414 crate::pr_outcome::FinalHead::At(&head),
19415 );
19416 assert!(
19417 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19418 );
19419 let omitted = posted(&mut f).await;
19421 assert!(omitted.body.contains("No description"));
19422 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19423 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19424 assert!(f.runner.state.change_summary.is_none());
19425 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19426 let described = posted(&mut f).await;
19427 assert!(described.body.contains("improve documentation for callers"));
19428 assert!(described.body.contains("remaining edge case"));
19429 assert!(described.body.contains("## Open review findings"));
19430 assert!(!described.body.contains("未修正"));
19431 }
19432 }
19433
19434 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19435 let mut f = summary_fixture();
19436 if !review_only {
19437 f.runner.state.candidates[0].agent = "alpha".into();
19438 }
19439 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19440 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19441 fixed
19442 .fix
19443 .as_mut()
19444 .unwrap()
19445 .rejected
19446 .push(crate::verdict::Rejection {
19447 id: "R1-1-1".into(),
19448 why: "呼び出されないコードのため辞退".into(),
19449 });
19450 let mut clean = clean_round_with_minor_findings();
19451 clean.verified_head = Some(head.clone());
19452 f.runner.state.reviews = vec![fixed, clean];
19453 ok_gate(&mut f.runner.state, Some(&head));
19454 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19455 f.runner.state.config.agents.push(scripted("beta", reply));
19456 f.runner.state.config.roles.language_judge =
19457 Some(crate::config::AgentChoice::One("beta".into()));
19458 f
19459 }
19460
19461 #[tokio::test]
19462 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19463 for review_only in [false, true] {
19464 let mut f = rewrite_fixture_with_findings(review_only,
19465 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19466 let m = posted(&mut f).await;
19467 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19468 let expected =
19469 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19470 assert!(m.body.contains(&expected));
19471 assert!(m.body.contains("## Review rounds"));
19472 assert!(
19473 m.body
19474 .contains("Gate (`verify.gate`): passed on the final commit")
19475 );
19476 assert!(
19477 m.body
19478 .contains("Tests (`verify.e2e`): passed on the final commit")
19479 );
19480 let c = f.runner.state.change_summary.as_ref().unwrap();
19481 assert!(
19482 m.body
19483 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19484 );
19485 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19486 assert!(crate::pr_outcome::covers_required_findings(
19487 &f.runner.state,
19488 &m.body
19489 ));
19490 assert!(m.body.contains("documentation remains incomplete"));
19491 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19492 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19493 assert!(f.runner.state.github_text.is_none());
19494 }
19495 }
19496
19497 #[tokio::test]
19498 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19499 for review_only in [false, true] {
19500 for reply in [
19501 "TITLE: fix: public description\nA short English description.",
19502 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19503 ] {
19504 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19505 let winner = f.runner.state.candidates[0].clone();
19506 let posted = f
19507 .runner
19508 .guarded_pr_message(&winner, None, true)
19509 .await
19510 .unwrap();
19511 assert!(posted.is_none());
19512 assert!(f.runner.state.parked);
19513 let withheld = f.runner.state.github_text.as_ref().unwrap();
19514 assert!(
19515 withheld.rewrite.is_none(),
19516 "an incomplete rewrite must not be selectable"
19517 );
19518 let q = ask::Questions::open()
19519 .get(withheld.question.as_ref().unwrap())
19520 .unwrap();
19521 assert!(
19522 !q.choices
19523 .contains(&crate::github_text::USE_REWRITE.to_owned())
19524 );
19525 assert!(
19526 q.detail.contains("R1-1-1")
19527 && q.detail.contains("R2-1-1")
19528 && q.detail.contains("R2-1-2")
19529 );
19530 }
19531 }
19532 }
19533
19534 #[tokio::test]
19535 async fn japanese_reports_stay_local_in_both_public_run_modes() {
19536 for review_only in [false, true] {
19537 let mut f = summary_fixture();
19538 if !review_only {
19539 f.runner.state.candidates[0].agent = "alpha".into();
19540 }
19541 f.runner.state.instruction = "内部タスクの説明".into();
19542 f.runner.state.candidates[0].summary = "内部の候補報告".into();
19543 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19544 f.runner.state.reviews = vec![
19545 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
19546 round_with_notes(2, Some("二回目の内部修正報告です。")),
19547 passing_round(3, Some(&head)),
19548 ];
19549 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
19550 ok_gate(&mut f.runner.state, Some(&head));
19551 let captured = f.repo.join("public-prompt.txt");
19552 let author = &mut f.runner.state.config.agents[0];
19553 author.command = vec![
19554 "sh".into(),
19555 "-c".into(),
19556 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19557 ];
19558 author.env.insert(
19559 "PROMPT_FILE".into(),
19560 captured.to_string_lossy().into_owned(),
19561 );
19562 let m = posted(&mut f).await;
19563 assert!(m.body.contains("Adds `own.txt`"));
19564 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
19565 assert!(m.body.contains("R1-1-1"));
19566 assert!(
19567 m.body
19568 .contains("Gate (`verify.gate`): passed on the final commit")
19569 );
19570 assert!(
19571 m.body
19572 .contains("Tests (`verify.e2e`): passed on the final commit")
19573 );
19574 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19575 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
19576 let prompt = std::fs::read_to_string(captured).unwrap();
19577 assert!(prompt.contains(&format!(
19578 "git diff {} {head}",
19579 f.runner.state.change_summary.as_ref().unwrap().base
19580 )));
19581 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
19582 assert_eq!(
19583 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
19584 "二回目の内部修正報告です。"
19585 );
19586 }
19587 }
19588
19589 #[tokio::test]
19590 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
19591 for review_only in [false, true] {
19592 let mut f = summary_fixture();
19593 if !review_only {
19594 f.runner.state.candidates[0].agent = "alpha".into();
19595 }
19596 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
19597 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
19598 std::fs::write(&f.reply, "no usable description").unwrap();
19599 let failed = posted(&mut f).await;
19600 assert!(
19601 failed
19602 .body
19603 .contains("No description of what this change does could be generated")
19604 );
19605 assert!(failed.body.is_ascii());
19606 assert!(f.runner.state.github_text.is_none());
19607 assert!(f.runner.state.change_summary.is_none());
19608 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19609 let evidence = crate::pr_outcome::summary_evidence(
19610 &f.runner.state,
19611 crate::pr_outcome::FinalHead::At(&head),
19612 );
19613 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
19614 assert!(
19615 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19616 .exists()
19617 );
19618 f.runner.state =
19619 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19620 std::fs::write(
19621 &f.reply,
19622 "TITLE: feat(own): add the own file\nAdds the committed own file.",
19623 )
19624 .unwrap();
19625 let retried = posted(&mut f).await;
19626 assert!(retried.body.contains("Adds the committed own file."));
19627 assert_eq!(
19628 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19629 "日本語の修正報告"
19630 );
19631 }
19632 }
19633
19634 #[tokio::test]
19635 async fn same_head_verification_and_findings_invalidate_public_prose() {
19636 let mut f = summary_fixture();
19637 posted(&mut f).await;
19638 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19639 let original = f
19640 .runner
19641 .state
19642 .change_summary
19643 .as_ref()
19644 .unwrap()
19645 .evidence
19646 .clone();
19647 ok_gate(&mut f.runner.state, Some(&head));
19648 std::fs::write(
19649 &f.reply,
19650 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19651 )
19652 .unwrap();
19653 let verified = posted(&mut f).await;
19654 assert!(verified.body.contains("now verified"));
19655 assert_ne!(
19656 f.runner.state.change_summary.as_ref().unwrap().evidence,
19657 original
19658 );
19659 let mut round = round_with_notes(1, Some("内部報告"));
19660 round.reviews.push(ReviewRecord {
19661 attempts: 0,
19662 reviewer: 1,
19663 agent: "alpha".into(),
19664 summary: "内部レビュー".into(),
19665 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19666 vote: None,
19667 failed: None,
19668 duration_ms: 0,
19669 });
19670 round
19671 .fix
19672 .as_mut()
19673 .unwrap()
19674 .rejected
19675 .push(crate::verdict::Rejection {
19676 id: "R1-1-1".into(),
19677 why: "呼び出し元から到達できない".into(),
19678 });
19679 f.runner.state.reviews = vec![round];
19680 let omitted = posted(&mut f).await;
19682 assert!(omitted.body.contains("No description"));
19683 assert!(omitted.body.contains("R1-1-1"));
19684 assert!(!omitted.body.contains("到達"));
19685 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19686 let described = posted(&mut f).await;
19687 assert!(described.body.contains("unreachable from callers"));
19688 assert!(described.body.contains("R1-1-1"));
19689 assert!(!described.body.contains("到達"));
19690 let evidence = f
19691 .runner
19692 .state
19693 .change_summary
19694 .as_ref()
19695 .unwrap()
19696 .evidence
19697 .clone();
19698 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19699 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19700 assert!(posted(&mut f).await.body.contains("rationale updated"));
19701 assert_ne!(
19702 f.runner.state.change_summary.as_ref().unwrap().evidence,
19703 evidence
19704 );
19705 f.runner
19707 .state
19708 .change_summary
19709 .as_mut()
19710 .unwrap()
19711 .evidence
19712 .clear();
19713 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19714 assert!(posted(&mut f).await.body.contains("current specification"));
19715 }
19716
19717 #[tokio::test]
19718 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19719 let mut f = summary_fixture();
19720 let m = posted(&mut f).await;
19721 assert_eq!(m.title, "feat(own): add the own file");
19722 assert!(
19723 m.body
19724 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19725 "{}",
19726 m.body
19727 );
19728 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19729 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19730 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19731 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19732 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19733 let kept = f.runner.state.change_summary.as_ref().unwrap();
19734 assert_eq!(
19735 (kept.head.as_str(), kept.base.as_str()),
19736 (head.as_str(), base.as_str())
19737 );
19738 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19739 }
19740
19741 #[tokio::test]
19742 async fn review_only_summary_receives_the_configured_github_language() {
19743 let mut f = summary_fixture();
19744 f.runner.state.config.graph.github_language = "ja".into();
19745 let captured = f.repo.join("summary-prompt.txt");
19746 let author = &mut f.runner.state.config.agents[0];
19747 author.command = vec![
19748 "sh".into(),
19749 "-c".into(),
19750 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19751 ];
19752 author.env.insert(
19753 "PROMPT_FILE".into(),
19754 captured.to_string_lossy().into_owned(),
19755 );
19756 std::fs::write(
19757 &f.reply,
19758 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19759 )
19760 .unwrap();
19761 let m = posted(&mut f).await;
19762 let prompt = std::fs::read_to_string(captured).unwrap();
19763 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19764 assert!(!prompt.contains("in English"));
19765 assert_eq!(m.title, "feat(own): ファイルを追加");
19766 assert!(m.body.contains("own.txt を追加します。"));
19767 }
19768
19769 #[tokio::test]
19770 async fn review_fixes_come_after_the_whole_change_summary() {
19771 let mut f = summary_fixture();
19772 f.runner.state.reviews = vec![
19773 round_with_notes(1, Some("handled the PATH case")),
19774 round_with_notes(2, None),
19775 ];
19776 let m = posted(&mut f).await;
19777 let summary = m.body.find("## Summary").expect("summary");
19778 let fixes = m.body.find("## Review rounds").expect("review rounds");
19779 assert!(summary < fixes, "{}", m.body);
19780 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19781 assert!(m.body.contains("## Final verification"), "{}", m.body);
19782 }
19783
19784 #[tokio::test]
19785 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19786 let mut f = summary_fixture();
19787 let first = posted(&mut f).await;
19788 let first_head = f.runner.state.change_summary.clone().unwrap().head;
19789 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19791 let again = posted(&mut f).await;
19792 assert_eq!((again.title, again.body), (first.title, first.body));
19793
19794 let g = |args: &[&str]| {
19795 let out = std::process::Command::new("git")
19796 .args(args)
19797 .current_dir(&f.repo)
19798 .quiet()
19799 .output()
19800 .expect("spawn git");
19801 assert!(out.status.success(), "git {args:?}");
19802 };
19803 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19804 g(&["add", "-A"]);
19805 g(&["commit", "-q", "-m", "fix(own): handle more"]);
19806 let second = posted(&mut f).await;
19807 let kept = f.runner.state.change_summary.clone().unwrap();
19808 assert_ne!(kept.head, first_head);
19809 assert!(second.body.contains("Not read yet."), "{}", second.body);
19810 assert!(!second.body.contains("so the first summary is read"));
19811 assert_eq!(second.title, "feat(own): other");
19812 }
19813
19814 #[tokio::test]
19815 async fn an_unavailable_summary_is_stated_and_never_invented() {
19816 let mut f = summary_fixture();
19817 std::fs::write(&f.reply, "no title line here").unwrap();
19818 let m = posted(&mut f).await;
19819 assert!(
19820 m.body
19821 .contains("No description of what this change does could be generated"),
19822 "{}",
19823 m.body
19824 );
19825 assert!(m.body.contains("internal reports remain in local records"));
19826 assert!(f.runner.state.change_summary.is_none());
19827 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19828 let winner = f.runner.state.candidates[0].clone();
19830 let preview = f
19831 .runner
19832 .guarded_pr_message(&winner, None, false)
19833 .await
19834 .unwrap()
19835 .unwrap();
19836 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19837 }
19838
19839 #[test]
19840 fn pr_message_review_single_commit_uses_its_subject() {
19841 let state = review_state(&["feat(nats): per-role user"]);
19842 let m = pr_message(&state, 'A');
19843 assert_eq!(m.title, "feat(nats): per-role user");
19844 assert!(!m.body.contains("Review the work already"), "{}", m.body);
19845 assert!(m.body.contains("## Commits under review"), "{}", m.body);
19846 }
19847
19848 #[test]
19849 fn pr_message_review_multi_commit_takes_the_oldest() {
19850 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19851 let m = pr_message(&state, 'A');
19852 assert_eq!(m.title, "feat: the change");
19853 for s in ["feat: the change", "fix: typo", "fix: again"] {
19854 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19855 }
19856 }
19857
19858 #[test]
19859 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19860 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19861 let mut state = review_state(&[first, "fix: later fixup"]);
19862 state.candidates[0].branch = "機能/ログイン".to_owned();
19863 let m = pr_message(&state, 'A');
19864 assert!(
19865 m.title.starts_with("chore: land candidate A of run"),
19866 "{}",
19867 m.title
19868 );
19869 }
19870 }
19871
19872 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19873 BranchFacts {
19874 commits: commits
19875 .iter()
19876 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19877 .collect(),
19878 stat: stat.to_owned(),
19879 }
19880 }
19881
19882 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19883 let mut r = review_round(true, 0, 1, 1, true, true);
19884 r.round = round;
19885 r.fix = notes.map(|n| FixRecord {
19886 agent: "fixer".to_owned(),
19887 addressed: Vec::new(),
19888 rejected: Vec::new(),
19889 notes: n.to_owned(),
19890 committed: true,
19891 failed: None,
19892 duration_ms: 0,
19893 continuation: None,
19894 });
19895 r
19896 }
19897
19898 #[test]
19899 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19900 let state = review_state(&["ignored"]);
19901 let f = facts(
19902 &[
19903 (
19904 "fix(login): resolve PATH on macOS",
19905 "Login shells skip rc files.",
19906 ),
19907 ("fix: address review", ""),
19908 ],
19909 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19910 );
19911 let m = pr_message_with(&state, 'A', Some(&f));
19912 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19913 assert!(
19914 m.body
19915 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
19916 "{}",
19917 m.body
19918 );
19919 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19920 assert!(m.body.contains("## Diff stat"), "{}", m.body);
19921 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19922 for banned in [
19923 "Review the work already",
19924 "no task statement",
19925 "Original task",
19926 ] {
19927 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19928 }
19929 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19930 }
19931
19932 #[test]
19933 fn pr_message_review_truncates_a_huge_first_commit_body() {
19934 let state = review_state(&["ignored"]);
19935 let f = facts(
19936 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19937 "s",
19938 );
19939 let m = pr_message_with(&state, 'A', Some(&f));
19940 assert!(m.body.len() < 30_000, "{}", m.body.len());
19941 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19942 assert!(
19943 m.body.contains("- ... 1 more commit(s)"),
19944 "{}",
19945 m.body.len()
19946 );
19947 assert!(m.body.ends_with("magi:candidate-a\n"));
19948 }
19949
19950 #[test]
19951 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19952 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19953 assert_eq!(m.title, "feat: x");
19954 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19955 assert!(!m.body.contains("Diff stat"), "{}", m.body);
19956 assert!(!m.body.contains("no task statement"), "{}", m.body);
19957 }
19958
19959 #[test]
19960 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19961 let mut state = review_state(&["日本語の件名"]);
19962 state.candidates[0].branch = "fix/macos-login-path".to_owned();
19963 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19964 state.candidates[0].branch = "機能/ログイン".to_owned();
19965 assert!(
19966 pr_message(&state, 'A')
19967 .title
19968 .starts_with("chore: land candidate A")
19969 );
19970 }
19971
19972 #[test]
19973 fn pr_message_review_fixes_survive_a_clean_final_round() {
19974 let mut state = review_state(&["feat: x"]);
19975 state.reviews = vec![
19976 round_with_notes(1, Some("handled the PATH case")),
19977 round_with_notes(2, None),
19978 ];
19979 let body = pr_message(&state, 'A').body;
19980 assert!(
19981 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19982 "{body}"
19983 );
19984 assert!(body.contains("### Round 1"), "{body}");
19985
19986 state.reviews = vec![
19987 round_with_notes(1, Some("first fix")),
19988 round_with_notes(2, Some("")),
19989 round_with_notes(3, Some("second fix")),
19990 round_with_notes(4, None),
19991 ];
19992 let body = pr_message(&state, 'A').body;
19993 assert!(
19994 body.contains("### Round 1") && !body.contains("first fix"),
19995 "{body}"
19996 );
19997 assert!(
19998 body.contains("### Round 3") && !body.contains("second fix"),
19999 "{body}"
20000 );
20001 assert!(body.contains("### Round 2"), "{body}");
20002 }
20003
20004 const FINAL: &str = "abcdef1234567890";
20005
20006 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20007 let mut r = round_with_notes(round, None);
20008 r.verified_head = verified.map(str::to_owned);
20009 r
20010 }
20011
20012 fn ok_gate(state: &mut RunState, head: Option<&str>) {
20013 state.gate = vec![CommandOutcome {
20014 command: "gate".to_owned(),
20015 code: Some(0),
20016 output_tail: String::new(),
20017 duration_ms: 0,
20018 resource_blocked: false,
20019 }];
20020 state.gate_ran = true;
20021 state.gate_head = head.map(str::to_owned);
20022 }
20023
20024 fn body_at(state: &RunState) -> String {
20025 pr_message_raw(
20026 state,
20027 'A',
20028 None,
20029 crate::pr_outcome::FinalHead::At(FINAL),
20030 ChangeNote::NotAsked,
20031 )
20032 .body
20033 }
20034
20035 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20036
20037 #[test]
20038 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20039 let mut state = review_state(&["feat: x"]);
20040 state.reviews = vec![
20041 round_with_notes(1, Some(SANDBOXED)),
20042 passing_round(2, Some(FINAL)),
20043 ];
20044 ok_gate(&mut state, Some(FINAL));
20045 let body = body_at(&state);
20046 assert!(!body.contains("Handled the PATH case."), "{body}");
20047 assert!(!body.contains("could not commit"), "{body}");
20048 assert!(!body.contains("could not run"), "{body}");
20049 assert!(
20050 body.contains("The fix is committed on the branch"),
20051 "{body}"
20052 );
20053 assert!(!body.contains("sandbox"), "{body}");
20054 assert!(
20055 body.contains("Tests (`verify.e2e`): passed on the final commit"),
20056 "{body}"
20057 );
20058 assert!(
20059 body.contains("Gate (`verify.gate`): passed on the final commit"),
20060 "{body}"
20061 );
20062 assert!(
20064 state.reviews[0]
20065 .fix
20066 .as_ref()
20067 .unwrap()
20068 .notes
20069 .contains("could not commit")
20070 );
20071 }
20072
20073 #[test]
20074 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20075 let mut state = review_state(&["feat: x"]);
20076 let mut r1 = round_with_notes(1, Some(SANDBOXED));
20077 r1.fix.as_mut().unwrap().committed = false;
20078 let mut r2 = passing_round(2, Some(FINAL));
20079 r2.e2e[0].code = Some(1);
20080 state.reviews = vec![r1, r2];
20081 let body = body_at(&state);
20082 assert!(!body.contains("could not commit"), "{body}");
20083 assert!(!body.contains("could not run"), "{body}");
20084 assert!(body.contains("This round produced no commit"), "{body}");
20085 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20086 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20087 assert!(!body.contains("passed"), "{body}");
20088 }
20089
20090 #[test]
20091 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20092 let mut state = review_state(&["feat: x"]);
20093 state.reviews = vec![
20094 round_with_notes(1, Some(SANDBOXED)),
20095 passing_round(2, Some("0123456789abcdef")),
20096 ];
20097 ok_gate(&mut state, Some("0123456789abcdef"));
20098 let body = body_at(&state);
20099 assert!(!body.contains("HTTP tests could not run"), "{body}");
20100 assert!(!body.contains("could not commit"), "{body}");
20102 assert!(
20103 body.contains("passed on an earlier commit (`0123456`)"),
20104 "{body}"
20105 );
20106 assert!(!body.contains("passed on the final commit"), "{body}");
20107
20108 ok_gate(&mut state, None);
20110 let body = body_at(&state);
20111 assert!(
20112 body.contains("passed on a commit that was not recorded"),
20113 "{body}"
20114 );
20115 }
20116
20117 #[test]
20118 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20119 let state = state_with_summary(
20120 "add retries to the client",
20121 "TITLE: feat: retries\n\nDid it.",
20122 );
20123 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20124 assert_eq!(m.title, "feat: retries");
20125 assert!(
20126 m.body.contains("<summary>Original task</summary>"),
20127 "{}",
20128 m.body
20129 );
20130 assert!(!m.body.contains("Commits under review"), "{}", m.body);
20131 assert!(
20132 m.body
20133 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20134 "{}",
20135 m.body
20136 );
20137 }
20138
20139 #[test]
20140 fn pr_message_review_bounds_a_long_english_subject() {
20141 let long = format!("feat: {}", "word ".repeat(100));
20142 let m = pr_message(&review_state(&[&long]), 'A');
20143 assert!(m.title.starts_with("feat: word"), "{}", m.title);
20144 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20145 }
20146
20147 #[test]
20148 fn pr_message_implementation_run_is_unchanged_by_review_support() {
20149 let state = state_with_summary("add retries\n\ndetails", "- did some things");
20150 let m = pr_message(&state, 'A');
20151 assert_eq!(m.title, "add retries");
20152 assert!(m.body.contains("<summary>Original task</summary>"));
20153 assert!(!m.body.contains("Commits under review"));
20154 assert_eq!(landing_subject_source(&state), state.instruction);
20155 }
20156
20157 #[test]
20158 fn review_run_squash_subject_is_the_change_not_the_prompt() {
20159 let state = review_state(&["feat: the change", "fix: typo"]);
20160 let source = landing_subject_source(&state);
20161 assert_eq!(land::merge_subject("", &source), "feat: the change");
20162 assert_eq!(
20163 land::merge_subject("magi: candidate A (uncommitted work)", &source),
20164 "feat: the change"
20165 );
20166 assert_eq!(
20168 land::merge_subject("feat: renamed by hand", &source),
20169 "feat: renamed by hand"
20170 );
20171 let blank = review_state(&["日本語"]);
20172 assert!(
20173 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20174 );
20175 }
20176
20177 #[test]
20178 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20179 let state = review_state(&["feat: the change"]);
20180 let source = landing_subject_source(&state);
20181 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20182 assert_eq!(
20183 land::merge_subject(landing_title(&state, old), &source),
20184 "feat: the change"
20185 );
20186 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20187 let task = state_with_summary("add retries", "");
20188 assert_eq!(landing_title(&task, old), old);
20189 }
20190
20191 #[test]
20192 fn pr_message_describes_the_change_not_the_task() {
20193 let state = state_with_summary(
20194 "今回やってほしいこと: results projector を直す",
20195 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20196 );
20197 let m = pr_message(&state, 'A');
20198 assert_eq!(m.title, "fix(web): batch the runs list reads");
20199 assert!(
20200 m.body.starts_with("## Summary\n\n- reads run.json once"),
20201 "{}",
20202 m.body
20203 );
20204 assert!(!m.body.contains("TITLE:"), "{}", m.body);
20205 let task_at = m.body.find("今回やってほしいこと").unwrap();
20206 let details_at = m.body.find("<details>").unwrap();
20207 assert!(
20208 details_at < task_at,
20209 "the task lives inside <details>: {}",
20210 m.body
20211 );
20212 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20213 assert!(m.body.contains("magi:candidate-a"));
20214 }
20215
20216 #[test]
20217 fn pr_message_falls_back_to_the_task_without_a_title_line() {
20218 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20219 let m = pr_message(&state, 'A');
20220 assert_eq!(m.title, "add retries");
20221 assert!(
20222 m.body.contains("## Summary\n\n- did some things"),
20223 "{}",
20224 m.body
20225 );
20226
20227 let none = RunState::new(
20228 PathBuf::from("/repo"),
20229 "main".to_owned(),
20230 "abc1234".to_owned(),
20231 "add retries".to_owned(),
20232 Config::default(),
20233 );
20234 let m = pr_message(&none, 'A');
20235 assert_eq!(m.title, "add retries");
20236 assert!(!m.body.contains("## Summary"), "{}", m.body);
20237 }
20238
20239 #[test]
20240 fn pr_message_refuses_the_candidate_commit_subject() {
20241 for bad in [
20242 "TITLE: magi: candidate A (uncommitted work)",
20243 "TITLE: chore: stuff (uncommitted work)",
20244 "TITLE: ",
20245 ] {
20246 let state = state_with_summary("add retries", bad);
20247 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20248 }
20249 }
20250
20251 #[test]
20252 fn pr_message_bounds_a_very_long_task_and_title() {
20253 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20254 let state = state_with_summary(&long, "- nothing");
20255 let m = pr_message(&state, 'A');
20256 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20257 assert!(!m.title.contains('\n'));
20258
20259 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20260 let m = pr_message(&state, 'A');
20261 assert!(m.title.starts_with("feat: "));
20262 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20263 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20264 }
20265
20266 fn long_title_of(instruction: &str) -> String {
20267 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20268 }
20269
20270 #[test]
20271 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20272 let first = "Make the landing path keep a readable title for long tasks";
20273 let line = format!(
20274 "{first}. {}",
20275 "And then keep going with more words ".repeat(20)
20276 );
20277 let t = long_title_of(&line);
20278 assert_eq!(t, first);
20279 assert!(!t.starts_with("chore: land"));
20280 }
20281
20282 #[test]
20283 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20284 let line = "word ".repeat(200);
20285 let t = long_title_of(&line);
20286 assert!(t.ends_with("word..."), "{t}");
20287 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20288 }
20289
20290 #[test]
20291 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20292 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20293 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20294 }
20295 }
20296
20297 #[test]
20298 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20299 let mut state = state_with_summary("再試行を追加する", "- nothing");
20300 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20301 state.config.graph.github_language = "ja".into();
20302 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20303 }
20304
20305 #[test]
20306 fn pr_message_title_limit_is_exact() {
20307 let at = "a".repeat(PR_TITLE_MAX);
20308 assert_eq!(long_title_of(&at), at);
20309 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20310 assert!(over.ends_with("..."), "{over}");
20311 assert_eq!(over.chars().count(), PR_TITLE_MAX);
20312 }
20313
20314 #[test]
20315 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20316 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20317 let t = long_title_of(&line);
20318 assert!(t.ends_with("..."), "{t}");
20319 assert!(t.is_ascii(), "{t}");
20320 }
20321
20322 #[test]
20323 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20324 let line = format!(
20325 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20326 "plus more filler words ".repeat(20)
20327 );
20328 let t = long_title_of(&line);
20329 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20330 }
20331
20332 #[test]
20333 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20334 let line = format!("\n\n# {}", "title words ".repeat(40));
20335 let state = state_with_summary(&line, "- nothing");
20336 let m = pr_message(&state, 'A');
20337 assert!(m.title.starts_with("title words"), "{}", m.title);
20338 assert_eq!(
20339 land::merge_subject(&m.title, &landing_subject_source(&state)),
20340 m.title
20341 );
20342 assert_eq!(
20344 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20345 "feat: renamed by hand"
20346 );
20347 }
20348
20349 #[tokio::test]
20350 async fn github_text_rewrite_is_bounded_and_falls_back() {
20351 crate::run::pin_test_home();
20352 for (reply, accepted) in [
20353 (
20354 "TITLE: fix: retries\nAdd retries for failed requests.",
20355 true,
20356 ),
20357 (
20358 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20359 false,
20360 ),
20361 (
20362 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20363 true,
20364 ),
20365 (
20366 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20367 true,
20368 ),
20369 (
20370 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20371 true,
20372 ),
20373 (
20374 "TITLE: fix: retries\n日本語の説明です。 token=secret",
20375 false,
20376 ),
20377 ("TITLE: fix: retries\n", false),
20378 ] {
20379 let dir = tempfile::tempdir().unwrap();
20380 let mut runner = runner_at(RunStatus::Gating);
20381 runner.state = state_with_summary(
20382 "add retries",
20383 "TITLE: fix: retries\n日本語の説明を書きます。",
20384 );
20385 runner.state.repo = dir.path().to_owned();
20386 runner.state.candidates[0].worktree = dir.path().to_owned();
20387 let mut author = spec("alpha");
20388 author.command = vec![
20389 "sh".into(),
20390 "-c".into(),
20391 "printf '%s' \"$REWRITE_REPLY\"".into(),
20392 ];
20393 author.env.insert("REWRITE_REPLY".into(), reply.into());
20394 runner.state.config.agents = vec![author];
20395 seed_public_summary(&mut runner);
20396 let winner = runner.state.candidates[0].clone();
20397 let first = runner
20398 .guarded_pr_message(&winner, None, true)
20399 .await
20400 .unwrap();
20401 let message = if accepted {
20402 first.expect("a passing rewrite never asks")
20403 } else {
20404 assert!(first.is_none());
20406 assert!(runner.state.parked);
20407 let store = ask::Questions::open();
20408 let asked: Vec<_> = store
20409 .list()
20410 .into_iter()
20411 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20412 .collect();
20413 assert_eq!(asked.len(), 1);
20414 assert!(!asked[0].detail.contains("secret"));
20415 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20416 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20417 assert_eq!(
20418 asked[0]
20419 .choices
20420 .contains(&crate::github_text::USE_REWRITE.to_owned()),
20421 rewrite.is_some()
20422 );
20423 store
20426 .update(&asked[0].id, |q| {
20427 q.abandon("test");
20428 Ok(())
20429 })
20430 .unwrap();
20431 let again = runner
20432 .guarded_pr_message(&winner, None, true)
20433 .await
20434 .unwrap()
20435 .expect("an abandoned question falls back");
20436 let n = store
20437 .list()
20438 .into_iter()
20439 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20440 .count();
20441 assert_eq!(n, 1, "asked once per rejected text");
20442 again
20443 };
20444 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20445 assert!(!message.body.contains("secret"));
20446 assert!(!message.body.contains("prefix"));
20447 assert!(!message.title.contains("secret"));
20448 assert_eq!(
20449 message.body.contains("Add retries for failed requests."),
20450 accepted
20451 );
20452 assert_eq!(
20453 runner.state.seats["github-text-rewrite"].turns, 1,
20454 "only one rewrite invocation"
20455 );
20456 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20457 if accepted {
20458 assert!(runner.state.github_text.is_none());
20459 assert!(!runner.state.parked);
20460 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20461 assert!(message.body.contains("## Final verification"));
20462 assert!(message.body.contains("Evidence: final diff"));
20463 assert!(
20464 message
20465 .body
20466 .contains(&format!("magi:run/{}", runner.state.id))
20467 );
20468 }
20469 if !accepted {
20470 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20471 assert!(
20472 message
20473 .body
20474 .contains(&format!("magi:run/{}", runner.state.id))
20475 );
20476 }
20477 }
20478 }
20479
20480 fn scripted(id: &str, reply: &str) -> AgentSpec {
20481 let mut a = spec(id);
20482 a.command = vec![
20483 "sh".into(),
20484 "-c".into(),
20485 "printf '%s' \"$REWRITE_REPLY\"".into(),
20486 ];
20487 a.env.insert("REWRITE_REPLY".into(), reply.into());
20488 a
20489 }
20490
20491 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20492 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20493
20494 fn seed_public_summary(runner: &mut Runner) {
20496 let repo = &runner.state.repo;
20497 let g = |args: &[&str]| {
20498 let out = std::process::Command::new("git")
20499 .args(args)
20500 .current_dir(repo)
20501 .quiet()
20502 .output()
20503 .unwrap();
20504 assert!(out.status.success(), "git {args:?}");
20505 String::from_utf8_lossy(&out.stdout).trim().to_owned()
20506 };
20507 g(&["init", "-q", "-b", "main"]);
20508 g(&[
20509 "-c",
20510 "user.name=Test",
20511 "-c",
20512 "user.email=test@example.invalid",
20513 "commit",
20514 "-q",
20515 "--allow-empty",
20516 "-m",
20517 "feat: fixture",
20518 ]);
20519 let head = g(&["rev-parse", "HEAD"]);
20520 g(&["update-ref", "refs/remotes/origin/main", &head]);
20521 let evidence = crate::pr_outcome::summary_evidence(
20522 &runner.state,
20523 crate::pr_outcome::FinalHead::At(&head),
20524 );
20525 let generated = &runner.state.candidates[0].summary;
20526 runner.state.change_summary = Some(crate::run::ChangeSummary {
20527 base: head.clone(),
20528 head,
20529 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
20530 title: summary_title(generated).unwrap(),
20531 body: summary_without_title(generated),
20532 });
20533 }
20534
20535 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
20536 let mut runner = runner_at(RunStatus::Gating);
20537 runner.state = state_with_summary("add retries", JA_SUMMARY);
20538 if review_only {
20539 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
20540 }
20541 runner.state.repo = dir.to_owned();
20542 runner.state.candidates[0].worktree = dir.to_owned();
20543 runner.state.config.agents = agents;
20544 seed_public_summary(&mut runner);
20545 runner
20546 }
20547
20548 fn gate_questions(runner: &Runner) -> usize {
20549 ask::Questions::open()
20550 .list()
20551 .into_iter()
20552 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20553 .count()
20554 }
20555
20556 #[tokio::test]
20557 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
20558 crate::run::pin_test_home();
20559 let dir = tempfile::tempdir().unwrap();
20560 let mut runner = rewrite_runner(
20561 dir.path(),
20562 true,
20563 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20564 );
20565 runner.state.config.roles.language_judge =
20566 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
20567 let winner = runner.state.candidates[0].clone();
20568 let msg = runner
20569 .guarded_pr_message(&winner, None, true)
20570 .await
20571 .unwrap()
20572 .expect("an English rewrite posts without asking");
20573 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
20574 assert!(msg.body.contains("<details>kept</details>"));
20575 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
20576 assert_eq!(gate_questions(&runner), 0);
20577 assert!(runner.state.github_text.is_none());
20578 assert!(!runner.state.seats.contains_key("impl-A"));
20579 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20580 }
20581
20582 #[tokio::test]
20583 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
20584 crate::run::pin_test_home();
20585 let dir = tempfile::tempdir().unwrap();
20586 let mut runner = rewrite_runner(
20587 dir.path(),
20588 false,
20589 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20590 );
20591 runner.state.config.roles.language_judge =
20592 Some(crate::config::AgentChoice::One("beta".into()));
20593 runner.state.seats.insert(
20594 "impl-A".into(),
20595 SeatState::new("impl-A", "alpha", runner.state.seed),
20596 );
20597 let winner = runner.state.candidates[0].clone();
20598 let msg = runner
20599 .guarded_pr_message(&winner, None, true)
20600 .await
20601 .unwrap()
20602 .expect("rewritten");
20603 assert!(msg.body.contains("<details>kept</details>"));
20604 assert_eq!(runner.state.seats["impl-A"].turns, 0);
20605 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20606 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20607 }
20608
20609 #[tokio::test]
20610 async fn rewrite_reuses_a_resumable_implementer_session() {
20611 crate::run::pin_test_home();
20612 let dir = tempfile::tempdir().unwrap();
20613 let mut runner = rewrite_runner(
20614 dir.path(),
20615 false,
20616 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20617 );
20618 runner.state.config.roles.language_judge =
20619 Some(crate::config::AgentChoice::One("beta".into()));
20620 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20621 seat.turns = 1;
20622 runner.state.seats.insert("impl-A".into(), seat);
20623 let winner = runner.state.candidates[0].clone();
20624 runner
20625 .guarded_pr_message(&winner, None, true)
20626 .await
20627 .unwrap()
20628 .expect("rewritten");
20629 assert_eq!(runner.state.seats["impl-A"].turns, 2);
20630 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20631 }
20632
20633 #[tokio::test]
20634 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20635 crate::run::pin_test_home();
20638 let dir = tempfile::tempdir().unwrap();
20639 let mut exhausted = spec("alpha");
20640 exhausted.command = vec!["false".into()];
20641 let mut runner = rewrite_runner(
20642 dir.path(),
20643 false,
20644 vec![exhausted, scripted("beta", EN_REWRITE)],
20645 );
20646 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20648 "alpha".into(),
20649 "beta".into(),
20650 ]));
20651 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20652 seat.turns = 1;
20653 runner.state.seats.insert("impl-A".into(), seat);
20654 let winner = runner.state.candidates[0].clone();
20655 let msg = runner
20656 .guarded_pr_message(&winner, None, true)
20657 .await
20658 .unwrap()
20659 .expect("the role rewrites after the implementer fails");
20660 assert!(msg.body.contains("`src/net.rs`"));
20661 assert_eq!(
20662 runner.state.seats["impl-A"].turns, 2,
20663 "the implementer was tried once"
20664 );
20665 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20666 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20667 }
20668
20669 #[tokio::test]
20670 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20671 crate::run::pin_test_home();
20672 let dir = tempfile::tempdir().unwrap();
20673 let mut broken = spec("bad");
20674 broken.command = vec!["false".into()];
20675 let mut runner = rewrite_runner(
20676 dir.path(),
20677 true,
20678 vec![
20679 broken,
20680 scripted("noformat", "Add retries."),
20681 scripted("good", EN_REWRITE),
20682 ],
20683 );
20684 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20685 "bad".into(),
20686 "noformat".into(),
20687 "good".into(),
20688 ]));
20689 let winner = runner.state.candidates[0].clone();
20690 let msg = runner
20691 .guarded_pr_message(&winner, None, true)
20692 .await
20693 .unwrap()
20694 .expect("the third agent rewrites");
20695 assert!(msg.body.contains("`src/net.rs`"));
20696 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20697 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20698 }
20699
20700 #[tokio::test]
20701 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20702 crate::run::pin_test_home();
20703 for (agents, role, saved) in [
20705 (
20706 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20707 Some("beta"),
20708 true,
20709 ),
20710 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20711 (Vec::new(), None, false),
20712 ] {
20713 let dir = tempfile::tempdir().unwrap();
20714 let mut runner = rewrite_runner(dir.path(), true, agents);
20715 runner.state.config.roles.language_judge =
20716 role.map(|r| crate::config::AgentChoice::One(r.into()));
20717 let winner = runner.state.candidates[0].clone();
20718 let first = runner
20719 .guarded_pr_message(&winner, None, true)
20720 .await
20721 .unwrap();
20722 assert!(first.is_none(), "withheld text parks the run");
20723 assert!(runner.state.parked);
20724 assert_eq!(gate_questions(&runner), 1);
20725 let g = runner.state.github_text.as_ref().unwrap();
20726 assert_eq!(g.rewrite.is_some(), saved);
20727 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20728 }
20729 }
20730
20731 #[tokio::test]
20732 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20733 crate::run::pin_test_home();
20734 let dir = tempfile::tempdir().unwrap();
20735 let mut runner = runner_at(RunStatus::Gating);
20736 runner.state =
20737 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20738 runner.state.repo = dir.path().to_owned();
20739 runner.state.candidates[0].worktree = dir.path().to_owned();
20740 let mut author = spec("alpha");
20741 author.command = vec![
20742 "sh".into(),
20743 "-c".into(),
20744 "printf '%s' \"$REWRITE_REPLY\"".into(),
20745 ];
20746 author.env.insert(
20747 "REWRITE_REPLY".into(),
20748 format!(
20749 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20750 "English sentence. ".repeat(600)
20751 ),
20752 );
20753 runner.state.config.agents = vec![author];
20754 seed_public_summary(&mut runner);
20755 let winner = runner.state.candidates[0].clone();
20756 assert!(
20757 runner
20758 .guarded_pr_message(&winner, None, true)
20759 .await
20760 .unwrap()
20761 .is_none()
20762 );
20763 let record = runner.state.github_text.clone().unwrap();
20764 let rewrite = record.rewrite.clone().unwrap();
20765 assert_eq!(rewrite.failed, ["body-language"]);
20766 assert!(
20767 rewrite
20768 .body
20769 .ends_with(&format!("magi:run/{}", runner.state.id))
20770 );
20771 let store = ask::Questions::open();
20772 let q = store.get(record.question.as_ref().unwrap()).unwrap();
20773 assert!(
20774 q.choices
20775 .contains(&crate::github_text::USE_REWRITE.to_owned())
20776 );
20777 for label in [
20778 "ORIGINAL",
20779 "FALLBACK",
20780 "REWRITE",
20781 "Rules still failed: body-language",
20782 "truncated",
20783 ] {
20784 assert!(q.detail.contains(label), "{label}");
20785 }
20786 let artifact = std::fs::read_to_string(crate::run::artifact_path(
20787 &runner.state,
20788 &crate::github_text::artifact_name(&record.fingerprint),
20789 ))
20790 .unwrap();
20791 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20792 runner.state =
20794 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20795 store
20796 .update(&q.id, |q| {
20797 q.say("まだ日本語")?;
20798 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20799 })
20800 .unwrap();
20801 assert!(
20802 runner
20803 .guarded_pr_message(&winner, None, true)
20804 .await
20805 .unwrap()
20806 .is_none()
20807 );
20808 let retry = runner.state.github_text.as_ref().unwrap();
20809 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20810 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20811 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20812 assert!(
20813 q.posting_comparison
20814 .as_ref()
20815 .unwrap()
20816 .matches_detail(&q.detail)
20817 );
20818 store
20819 .update(&q.id, |q| {
20820 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20821 })
20822 .unwrap();
20823 let message = runner
20824 .guarded_pr_message(&winner, None, true)
20825 .await
20826 .unwrap()
20827 .unwrap();
20828 assert_eq!(message.title, rewrite.title);
20829 assert_eq!(message.body, rewrite.body);
20830 assert_eq!(
20831 runner.state.github_text.as_ref().unwrap().choice,
20832 Some(GithubTextChoice::Rewrite)
20833 );
20834 let again = runner
20835 .guarded_pr_message(&winner, None, true)
20836 .await
20837 .unwrap()
20838 .unwrap();
20839 assert_eq!(again.title, message.title);
20840 assert_eq!(again.body, message.body);
20841 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20842 assert!(
20843 !runner
20844 .state
20845 .events
20846 .iter()
20847 .any(|e| e.message.contains(&rewrite.title)
20848 || e.message.contains("English sentence."))
20849 );
20850 }
20851
20852 #[tokio::test]
20853 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20854 crate::run::pin_test_home();
20855 let dir = tempfile::tempdir().unwrap();
20856 let mut runner = runner_at(RunStatus::Gating);
20857 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20858 runner.state.repo = dir.path().to_owned();
20859 runner.state.candidates[0].worktree = dir.path().to_owned();
20860 seed_public_summary(&mut runner);
20861 let winner = runner.state.candidates[0].clone();
20862 assert!(
20863 runner
20864 .guarded_pr_message(&winner, None, true)
20865 .await
20866 .unwrap()
20867 .is_none()
20868 );
20869 let record = runner.state.github_text.as_ref().unwrap();
20870 assert!(record.rewrite.is_none());
20871 let mut old = serde_json::to_value(record).unwrap();
20873 old.as_object_mut().unwrap().remove("rewrite");
20874 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20875 assert!(old.rewrite.is_none());
20876 let id = record.question.clone().unwrap();
20877 ask::Questions::open()
20878 .update(&id, |q| {
20879 q.choices.push(crate::github_text::USE_REWRITE.into());
20880 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20881 })
20882 .unwrap();
20883 let message = runner
20884 .guarded_pr_message(&winner, None, true)
20885 .await
20886 .unwrap()
20887 .unwrap();
20888 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20889 assert_eq!(
20890 runner.state.github_text.as_ref().unwrap().choice,
20891 Some(GithubTextChoice::Fallback)
20892 );
20893 }
20894
20895 #[tokio::test]
20896 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20897 crate::run::pin_test_home();
20898 let dir = tempfile::tempdir().unwrap();
20899 let mut runner = runner_at(RunStatus::Gating);
20900 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20901 runner.state.repo = dir.path().to_owned();
20902 runner.state.candidates[0].worktree = dir.path().to_owned();
20903 seed_public_summary(&mut runner);
20904 let winner = runner.state.candidates[0].clone();
20905 assert!(
20906 runner
20907 .guarded_pr_message(&winner, None, true)
20908 .await
20909 .unwrap()
20910 .is_none()
20911 );
20912 let store = ask::Questions::open();
20913 let find = |run: &str| {
20914 store
20915 .list()
20916 .into_iter()
20917 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20918 .unwrap()
20919 };
20920 let run = runner.state.id.clone();
20921 let q = find(&run);
20922 let fp = runner
20923 .state
20924 .github_text
20925 .as_ref()
20926 .unwrap()
20927 .fingerprint
20928 .clone();
20929 let name = crate::github_text::artifact_name(&fp);
20930 let path = crate::run::artifact_path(&runner.state, &name);
20931 assert!(q.detail.contains(&path.display().to_string()));
20932 assert!(q.detail.contains("title-language: title"));
20933 let saved = std::fs::read_to_string(&path).expect("artifact written");
20934 assert!(saved.contains("日本語のタイトル"));
20935 assert!(q.cwd.is_none());
20936 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20937 let say = |id: &str, text: &str, choice: &str| {
20939 store
20940 .update(id, |q| {
20941 q.thread.push(ask::Turn {
20942 who: ask::Who::Operator,
20943 body: text.to_owned(),
20944 at: jiff::Timestamp::now(),
20945 note: None,
20946 });
20947 q.answer(ask::Answer::Choice(choice.to_owned()))
20948 })
20949 .unwrap();
20950 };
20951 say(&q.id, "まだ日本語", "use my text");
20952 assert!(
20953 runner
20954 .guarded_pr_message(&winner, None, true)
20955 .await
20956 .unwrap()
20957 .is_none()
20958 );
20959 let second = find(&run);
20960 assert_ne!(second.id, q.id);
20961 assert!(second.detail.contains("did not pass"));
20962 assert!(second.detail.contains("日本語のタイトル"));
20965 assert!(!second.detail.contains("まだ日本語"));
20966 assert!(second.detail.contains(&path.display().to_string()));
20967 say(&second.id, "fix: retry failed requests", "use my text");
20968 let message = runner
20969 .guarded_pr_message(&winner, None, true)
20970 .await
20971 .unwrap()
20972 .expect("a vetted replacement is posted");
20973 assert_eq!(message.title, "fix: retry failed requests");
20974 assert_eq!(
20975 runner
20976 .state
20977 .github_text
20978 .as_ref()
20979 .unwrap()
20980 .chosen_title
20981 .as_deref(),
20982 Some("fix: retry failed requests")
20983 );
20984 assert!(runner.state.github_text.as_ref().unwrap().resolved);
20985 let again = runner
20987 .guarded_pr_message(&winner, None, true)
20988 .await
20989 .unwrap()
20990 .unwrap();
20991 assert_eq!(again.title, message.title);
20992 assert_eq!(
20993 store
20994 .list()
20995 .into_iter()
20996 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
20997 .count(),
20998 2
20999 );
21000 }
21001
21002 #[test]
21003 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21004 let mut state = state_with_summary(
21008 "add retries",
21009 "TITLE: fix(web): batch reads\n- reads run.json once",
21010 );
21011 state.config.graph.language = "ja".to_owned();
21012 let m = pr_message(&state, 'A');
21013 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21014 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21015
21016 let task = "今回やってほしいこと: results projector を直す";
21019 let mut state = state_with_summary(task, "- no title line");
21020 state.config.graph.language = "ja".to_owned();
21021 let m = pr_message(&state, 'A');
21022 assert_eq!(
21023 m.title,
21024 format!("chore: land candidate A of run {}", state.id)
21025 );
21026 assert!(
21027 m.body.contains(&format!(
21028 "<summary>Original task</summary>\n\n{task}\n\n</details>"
21029 )),
21030 "{}",
21031 m.body
21032 );
21033 }
21034
21035 #[test]
21036 fn pr_message_scrubs_home_paths_and_addresses() {
21037 let state = state_with_summary(
21038 "fix it in /Users/someone/src/x",
21039 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21040 );
21041 let m = pr_message(&state, 'A');
21042 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21043 assert!(!m.body.contains(leak), "{}", m.body);
21044 }
21045 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21046 }
21047
21048 #[test]
21049 fn pr_message_survives_a_task_that_closes_details() {
21050 let state = state_with_summary("a </details> b", "TITLE: fix: x");
21051 let m = pr_message(&state, 'A');
21052 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21053 }
21054
21055 #[test]
21056 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21057 let cmd = manual_merge_command(
21058 MergeStyle::Squash,
21059 Path::new("/repo"),
21060 "b",
21061 "fix: \"quoted\" $(x) `y`\n\nbody",
21062 );
21063 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21064 }
21065
21066 #[test]
21067 fn manual_merge_command_matches_the_configured_style() {
21068 let repo = Path::new("/repo");
21069 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21070
21071 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21072 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21073
21074 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21075 assert_eq!(
21076 squash,
21077 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21078 \"Merge magi run 0832 (candidate A)\""
21079 );
21080
21081 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21082 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21083 }
21084
21085 #[test]
21086 fn a_nudge_gets_a_quarter_of_the_budget() {
21087 assert_eq!(retry_budget(secs(1200), true), secs(300));
21089 assert_eq!(retry_budget(secs(3600), true), secs(900));
21090 }
21091
21092 #[test]
21093 fn a_resent_prompt_keeps_the_whole_budget() {
21094 assert_eq!(retry_budget(secs(1200), false), secs(1200));
21097 assert_eq!(retry_budget(secs(60), false), secs(60));
21098 }
21099
21100 #[test]
21101 fn the_floor_never_exceeds_the_original_budget() {
21102 assert_eq!(retry_budget(secs(60), true), secs(60));
21106 assert_eq!(retry_budget(secs(480), true), secs(120));
21107 assert_eq!(retry_budget(secs(0), true), secs(0));
21108 }
21109
21110 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21111 agent::CommandEvidence {
21112 id: "item1".to_owned(),
21113 description: "cargo test".to_owned(),
21114 exit_code,
21115 result_summary: String::new(),
21116 source: "codex".to_owned(),
21117 }
21118 }
21119
21120 #[test]
21121 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21122 assert!(!has_unconfirmed_command(&[]));
21126 }
21127
21128 #[test]
21129 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21130 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21134 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21135 assert!(!has_unconfirmed_command(&[
21136 evidence(Some(0)),
21137 evidence(Some(101))
21138 ]));
21139 }
21140
21141 #[test]
21142 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21143 assert!(has_unconfirmed_command(&[
21144 evidence(Some(0)),
21145 evidence(None)
21146 ]));
21147 }
21148
21149 #[test]
21150 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21151 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21152 assert_eq!(
21153 verified_noop_claim(true, &[], text).as_deref(),
21154 Some("already fixed by b32cfc4, on main.")
21155 );
21156 }
21157
21158 #[test]
21159 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21160 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21163 assert!(verified_noop_claim(false, &[], text).is_none());
21164 }
21165
21166 #[test]
21167 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21168 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21169 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21170 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21172 }
21173
21174 #[test]
21175 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21176 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21177 }
21178
21179 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21182 runner.state.candidates = shape
21183 .iter()
21184 .enumerate()
21185 .map(|(i, &(empty, verified))| Candidate {
21186 index: i,
21187 label: (b'A' + i as u8) as char,
21188 agent: "sonnet".to_owned(),
21189 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21190 worktree: PathBuf::from(format!("/wt/{i}")),
21191 summary: String::new(),
21192 stat: String::new(),
21193 files: 0,
21194 commits: 0,
21195 empty,
21196 failed: None,
21197 verified_noop: verified.map(str::to_owned),
21198 duration_ms: 0,
21199 folded: false,
21200 })
21201 .collect();
21202 }
21203
21204 #[test]
21205 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21206 ask_test_home();
21207 let mut runner = runner_at(RunStatus::Implementing);
21208 set_candidates(
21209 &mut runner,
21210 &[
21211 (true, Some("already on main at b32cfc4")),
21212 (true, Some("same fix, see the existing test")),
21213 ],
21214 );
21215
21216 runner
21217 .after_implement()
21218 .expect("a verified no-op is not an error");
21219
21220 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21221 }
21222
21223 #[test]
21224 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21225 ask_test_home();
21226 let mut runner = runner_at(RunStatus::Implementing);
21227 set_candidates(
21231 &mut runner,
21232 &[(true, Some("already on main at b32cfc4")), (true, None)],
21233 );
21234
21235 let err = runner
21236 .after_implement()
21237 .expect_err("an unverified empty candidate must still fail the run");
21238
21239 assert!(
21240 err.to_string().contains("no candidate produced a change"),
21241 "{err}"
21242 );
21243 assert_eq!(runner.state.status, RunStatus::Failed);
21244 }
21245
21246 #[test]
21247 fn after_implement_still_fails_an_ordinary_all_empty_run() {
21248 ask_test_home();
21249 let mut runner = runner_at(RunStatus::Implementing);
21250 set_candidates(&mut runner, &[(true, None), (true, None)]);
21251
21252 let err = runner
21253 .after_implement()
21254 .expect_err("no candidate declared anything; this is an ordinary failure");
21255
21256 assert!(
21257 err.to_string().contains("no candidate produced a change"),
21258 "{err}"
21259 );
21260 assert_eq!(runner.state.status, RunStatus::Failed);
21261 }
21262}
21263
21264#[derive(Debug)]
21267struct GraftRestoreFailed(String);
21268
21269impl std::fmt::Display for GraftRestoreFailed {
21270 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21271 write!(
21272 f,
21273 "could not restore the winner to its pre-graft commit: {}",
21274 self.0
21275 )
21276 }
21277}
21278
21279impl std::error::Error for GraftRestoreFailed {}
21280
21281#[cfg(test)]
21285mod resource_wait_tests {
21286 use super::*;
21287 use std::collections::BTreeMap;
21288
21289 fn sh(id: &str, script: &str) -> AgentSpec {
21290 AgentSpec {
21291 id: id.to_owned(),
21292 kind: crate::config::AgentKind::Command,
21293 model: None,
21294 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21295 extra_args: Vec::new(),
21296 env: BTreeMap::new(),
21297 prompt_delivery: None,
21298 max_turn_secs: None,
21299 }
21300 }
21301
21302 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21303 SeatJob {
21304 seat: SeatState::new("impl-A", &spec.id, 1),
21305 spec,
21306 cwd: dir.to_path_buf(),
21307 prompt: "do it".to_owned(),
21308 timeout,
21309 allow_write: true,
21310 sessions: false,
21311 artifacts: dir.join("art"),
21312 stem: "impl-A".to_owned(),
21313 handover: None,
21314 }
21315 }
21316
21317 fn state(resource_wait_secs: u64) -> RunState {
21318 let mut config = Config::default();
21319 config.graph.timeout_resource_wait = resource_wait_secs;
21320 RunState::new(
21321 PathBuf::from("/repo"),
21322 "main".to_owned(),
21323 "abc1234".to_owned(),
21324 "task".to_owned(),
21325 config,
21326 )
21327 }
21328
21329 fn hold(cache: &Path) -> crate::cache::Guard {
21330 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21331 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21332 crate::cache::AcquireOutcome::Acquired(g) => g,
21333 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21334 }
21335 }
21336
21337 async fn run(
21338 jobs: Vec<SeatJob>,
21339 sem: Arc<Semaphore>,
21340 cache: Option<&Path>,
21341 st: &mut RunState,
21342 ) -> Vec<(usize, SeatState, AgentOutcome)> {
21343 let prompts = Prompts::default();
21344 let ctx = WaveCtx {
21345 run: "r",
21346 node: "implement",
21347 prompts: &prompts,
21348 cache,
21349 round: None,
21350 carry_seats: false,
21351 pauses: None,
21352 };
21353 wave(jobs, sem, &ctx, st, 0).await
21354 }
21355
21356 #[tokio::test]
21357 async fn verification_cache_wait_preserves_the_command_budget() {
21358 crate::run::pin_test_home();
21359 let tmp = tempfile::tempdir().unwrap();
21360 let cache = tmp.path().join("target");
21361 let guard = hold(&cache);
21362 let release = tokio::spawn(async move {
21363 tokio::time::sleep(Duration::from_millis(100)).await;
21364 guard.release();
21365 });
21366 let mut st = state(10);
21367 let budget = Duration::from_millis(50);
21368 let wait = repository_wait(&st, None);
21369 let (outcomes, _) = with_cache_lease(
21370 &mut st,
21371 Some(&cache),
21372 "gate",
21373 "gate",
21374 tmp.path(),
21375 "abc1234",
21376 budget,
21377 "test gate",
21378 &wait,
21379 |_, available| async move {
21380 assert_eq!(
21381 available, budget,
21382 "contention must not shorten verification"
21383 );
21384 (Vec::new(), false, Vec::new())
21385 },
21386 )
21387 .await
21388 .unwrap();
21389 release.await.unwrap();
21390 assert!(outcomes.is_empty(), "{outcomes:?}");
21391 assert!(
21392 st.events
21393 .iter()
21394 .any(|event| event.message.contains("waiting for"))
21395 );
21396 assert!(st.resource_wait.is_none());
21397 hold(&cache).release();
21399 }
21400
21401 #[tokio::test]
21402 async fn verification_cache_wait_expires_without_starting_commands() {
21403 crate::run::pin_test_home();
21404 let tmp = tempfile::tempdir().unwrap();
21405 let cache = tmp.path().join("target");
21406 let _held = hold(&cache);
21407 let mut st = state(0);
21408 let wait = repository_wait(&st, None);
21409 let (outcomes, _) = with_cache_lease(
21410 &mut st,
21411 Some(&cache),
21412 "gate",
21413 "gate",
21414 tmp.path(),
21415 "abc1234",
21416 Duration::from_secs(60),
21417 "test gate",
21418 &wait,
21419 |_, _| async { panic!("commands must not start without the cache lease") },
21420 )
21421 .await
21422 .unwrap();
21423 assert_eq!(outcomes.len(), 1);
21424 assert!(outcomes[0].resource_blocked);
21425 assert!(st.resource_wait.is_none());
21426 assert!(matches!(
21427 st.resource_block.unwrap().cause,
21428 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21429 ));
21430 }
21431
21432 #[tokio::test]
21433 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21434 crate::run::pin_test_home();
21435 let tmp = tempfile::tempdir().unwrap();
21436 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21437 let cache = tmp.path().join("cache");
21438 let mut state = state(21600);
21439 let wait = repository_wait(&state, None);
21440 let (outcomes, _) = with_cache_lease(
21441 &mut state,
21442 Some(&cache),
21443 "gate",
21444 "gate",
21445 tmp.path(),
21446 "head",
21447 Duration::from_secs(60),
21448 "gate",
21449 &wait,
21450 |_, _| async { panic!("freshness failure must not start verification") },
21451 )
21452 .await
21453 .unwrap();
21454 assert_eq!(outcomes.len(), 1);
21455 assert!(outcomes[0].resource_blocked);
21456 let block = state.resource_block.unwrap();
21457 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21458 assert!(!block.description().contains("timed out"));
21459 }
21460
21461 #[tokio::test]
21462 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21463 crate::run::pin_test_home();
21464 let tmp = tempfile::tempdir().unwrap();
21465 let cache = tmp.path().join("cache");
21466 let _held = hold(&cache);
21467 let mut state = state(21600);
21468 let pause = Pause::new();
21469 let interrupt = Pause::new();
21470 let cause = ParkCause::Operator;
21471 pause.park_for(cause.clone());
21472 let prompts = Prompts::default();
21473 let ctx = WaveCtx {
21474 run: "test",
21475 node: "fix",
21476 prompts: &prompts,
21477 cache: Some(&cache),
21478 round: None,
21479 carry_seats: false,
21480 pauses: Some((&pause, &interrupt)),
21481 };
21482 let result = wave(
21483 vec![job(
21484 sh("writer", "exit 99"),
21485 tmp.path(),
21486 Duration::from_secs(1),
21487 )],
21488 Arc::new(Semaphore::new(1)),
21489 &ctx,
21490 &mut state,
21491 0,
21492 )
21493 .await;
21494 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21495 assert_eq!(result[0].1.turns, 0);
21496 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21497 assert_eq!(
21498 state.resource_block.unwrap().cause,
21499 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21500 );
21501 assert!(state.active.is_empty());
21502 }
21503
21504 #[tokio::test]
21505 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21506 crate::run::pin_test_home();
21507 let tmp = tempfile::tempdir().unwrap();
21508 let cache = tmp.path().join("target-a");
21509 let guard = hold(&cache);
21510 tokio::spawn(async move {
21512 tokio::time::sleep(Duration::from_millis(1500)).await;
21513 guard.release();
21514 });
21515 let mut st = state(60);
21516 let results = run(
21517 vec![job(
21518 sh("alpha", "echo ok"),
21519 tmp.path(),
21520 Duration::from_secs(1),
21521 )],
21522 Arc::new(Semaphore::new(1)),
21523 Some(&cache),
21524 &mut st,
21525 )
21526 .await;
21527 assert!(
21528 matches!(results[0].2, AgentOutcome::Ok(_)),
21529 "the wait must not be charged to the agent"
21530 );
21531 assert!(st.resource_block.is_none());
21532 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21533 assert!(st.resource_wait.is_none() && st.active.is_empty());
21534 }
21535
21536 #[tokio::test]
21537 #[cfg(unix)]
21538 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
21539 use std::os::fd::AsRawFd as _;
21540 crate::run::pin_test_home();
21541 let tmp = tempfile::tempdir().unwrap();
21542 crate::git::git(tmp.path(), &["init"]).await.unwrap();
21543 let common = tmp.path().join(".git");
21544 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
21545 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
21546 assert_eq!(
21547 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
21548 0
21549 );
21550 let mut st = state(1);
21551 st.config.coordination = crate::config::test_coordination();
21552 let mut reader = job(
21553 sh("reader", "echo must-not-run"),
21554 tmp.path(),
21555 Duration::from_secs(30),
21556 );
21557 reader.allow_write = false;
21558 let results = run(
21559 vec![reader.clone()],
21560 Arc::new(Semaphore::new(1)),
21561 None,
21562 &mut st,
21563 )
21564 .await;
21565 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21566 assert_eq!(results[0].1.turns, 0);
21567 assert!(FailClass::of(&results[0].2).is_none());
21568 assert_eq!(
21569 st.resource_block.as_ref().unwrap().kind,
21570 "repository coordination"
21571 );
21572 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21573 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21574 drop(external);
21575 let editor = crate::repo_lock::SeatLease::for_checkout(
21576 tmp.path(),
21577 true,
21578 crate::config::test_coordination().as_ref(),
21579 )
21580 .await
21581 .unwrap();
21582 let pause = Pause::new();
21583 let interrupt = Pause::new();
21584 let prompts = Prompts::default();
21585 let ctx = WaveCtx {
21586 run: "r",
21587 node: "review",
21588 prompts: &prompts,
21589 cache: None,
21590 round: None,
21591 carry_seats: false,
21592 pauses: Some((&pause, &interrupt)),
21593 };
21594 let (results, ()) = tokio::join!(
21595 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
21596 async {
21597 tokio::time::sleep(Duration::from_millis(50)).await;
21598 pause.park();
21599 }
21600 );
21601 assert!(
21602 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
21603 );
21604 assert_eq!(results[0].1.turns, 0);
21605 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21606 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21607 drop(editor);
21608 }
21609
21610 #[tokio::test]
21611 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
21612 crate::run::pin_test_home();
21613 let tmp = tempfile::tempdir().unwrap();
21614 let cache = tmp.path().join("target-b");
21615 let _held = hold(&cache);
21616 let mut st = state(1);
21617 let results = run(
21618 vec![job(
21619 sh("alpha", "echo ok"),
21620 tmp.path(),
21621 Duration::from_secs(1),
21622 )],
21623 Arc::new(Semaphore::new(1)),
21624 Some(&cache),
21625 &mut st,
21626 )
21627 .await;
21628 let (_, seat, out) = &results[0];
21629 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21630 assert!(wave_blocked(&results));
21631 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21632 assert_eq!(seat.turns, 0, "nothing was asked");
21633 assert!(
21634 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21635 "no agent was spawned"
21636 );
21637 assert!(FailClass::of(out).is_none(), "not a failure class");
21638 let block = st.resource_block.as_ref().expect("recorded for the resume");
21639 assert_eq!(block.kind, "shared build cache");
21640 assert!(matches!(
21641 block.cause,
21642 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21643 ));
21644 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21645 assert!(st.active.is_empty());
21646 }
21647
21648 #[tokio::test]
21649 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21650 crate::run::pin_test_home();
21651 let tmp = tempfile::tempdir().unwrap();
21652 let mut st = state(1);
21653 let results = run(
21654 vec![job(
21655 sh("alpha", "echo ok"),
21656 tmp.path(),
21657 Duration::from_secs(1),
21658 )],
21659 Arc::new(Semaphore::new(0)),
21660 None,
21661 &mut st,
21662 )
21663 .await;
21664 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21665 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21666 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21667 }
21668
21669 #[tokio::test]
21670 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21671 crate::run::pin_test_home();
21672 let tmp = tempfile::tempdir().unwrap();
21673 let mut st = state(60);
21674 let results = run(
21675 vec![job(
21676 sh("alpha", "sleep 30"),
21677 tmp.path(),
21678 Duration::from_secs(1),
21679 )],
21680 Arc::new(Semaphore::new(1)),
21681 None,
21682 &mut st,
21683 )
21684 .await;
21685 let out = &results[0].2;
21686 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21687 let class = FailClass::of(out).expect("a failure");
21688 assert_eq!(class, FailClass::Timeout);
21689 assert!(should_hand_over(None, &class), "existing handover policy");
21690 assert!(st.resource_block.is_none());
21691 }
21692}