1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118const MAX_FIX_CONTINUATIONS: usize = 2;
134
135#[derive(Clone)]
141struct SeatJob {
142 spec: AgentSpec,
143 seat: SeatState,
144 cwd: PathBuf,
145 prompt: String,
146 timeout: Duration,
147 allow_write: bool,
148 sessions: bool,
149 artifacts: PathBuf,
150 stem: String,
151 handover: Option<String>,
156}
157
158enum AgentOutcome {
170 Ok(AgentOutput),
172 Quota(AgentOutput),
174 Dropped(AgentOutput),
177 Failed(String),
179 ResourceBlocked(String),
186}
187
188#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218 #[must_use]
220 pub fn new() -> Self {
221 Self::default()
222 }
223
224 pub fn park(&self) {
226 self.0.store(true, Ordering::SeqCst);
227 }
228
229 pub fn park_because(&self, reason: impl Into<String>) {
235 self.park_for(ParkCause::Interrupt {
236 reason: reason.into(),
237 });
238 }
239
240 pub fn park_for(&self, cause: ParkCause) {
243 let mut reason_guard = self
244 .1
245 .lock()
246 .unwrap_or_else(std::sync::PoisonError::into_inner);
247 if reason_guard.is_none() {
248 *reason_guard = Some(cause);
249 }
250 drop(reason_guard);
251 self.park();
252 }
253
254 #[must_use]
256 pub fn parked(&self) -> bool {
257 self.0.load(Ordering::SeqCst)
258 }
259
260 #[must_use]
262 pub fn reason(&self) -> Option<String> {
263 self.cause().map(|c| c.label())
264 }
265
266 #[must_use]
268 pub fn cause(&self) -> Option<ParkCause> {
269 self.1
270 .lock()
271 .unwrap_or_else(std::sync::PoisonError::into_inner)
272 .clone()
273 }
274}
275
276pub struct Runner {
278 pub state: RunState,
280 roles: ResolvedRoles,
281 sem: Arc<Semaphore>,
282 pause: Pause,
286 interrupt: Pause,
292}
293
294async fn review_base(
299 repo: &Path,
300 remote: &str,
301 base_branch: &str,
302 base_commit: &str,
303 branch: &str,
304) -> String {
305 review_base_checked(repo, remote, base_branch, base_commit, branch)
306 .await
307 .0
308}
309
310async fn review_base_checked(
315 repo: &Path,
316 remote: &str,
317 base_branch: &str,
318 base_commit: &str,
319 branch: &str,
320) -> (String, bool) {
321 let tracking = format!("{remote}/{base_branch}");
322 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323 if git::rev_exists(repo, &tracking).await
324 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325 && !mb.is_empty()
326 {
327 return (mb, fresh);
328 }
329 let mb = git::merge_base(repo, base_commit, branch)
330 .await
331 .ok()
332 .filter(|mb| !mb.is_empty())
333 .unwrap_or_else(|| base_commit.to_owned());
334 (mb, false)
335}
336
337pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341 if !is_review_run(state) {
342 return;
343 }
344 let base = review_base(
345 &state.repo,
346 &state.config.merge.remote,
347 &state.base_branch,
348 &state.base_commit,
349 branch,
350 )
351 .await;
352 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353 && !subjects.is_empty()
354 && state.reviewed_commits.as_ref() != Some(&subjects)
355 {
356 state.reviewed_commits = Some(subjects);
357 state.save().ok();
358 }
359}
360
361async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365 let (base, trusted) = review_base_checked(
366 &state.repo,
367 &state.config.merge.remote,
368 &state.base_branch,
369 &state.base_commit,
370 branch,
371 )
372 .await;
373 if !trusted {
374 return None;
375 }
376 git::subjects(&state.repo, &state.base_commit, &base)
377 .await
378 .ok()
379}
380
381pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386 if is_leak(computed) {
387 return false;
388 }
389 let cur = current.trim();
390 cur.is_empty()
391 || cur.starts_with(REVIEW_PROMPT_OPENING)
392 || cur.starts_with("chore: land candidate")
393 || cur.starts_with("magi: candidate")
394 || is_leak(cur)
395}
396
397async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426 let tracking = format!("{remote}/{branch}");
427 let fetched = git::fetch(repo, remote, branch).await;
428 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429 let local_exists = git::branch_exists(repo, branch).await?;
430 if !fresh {
431 if !local_exists {
432 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433 }
434 tracing::warn!(
435 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436 );
437 return Ok(());
438 }
439 let remote_sha = git::rev_parse(repo, &tracking).await?;
440 if !local_exists {
441 git::git(repo, &["branch", branch, &tracking]).await?;
442 return Ok(());
443 }
444 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446 return Ok(());
447 }
448 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455 .await?
456 {
457 crate::reconcile::Reconciliation::Pushed => {
458 tracing::warn!(
459 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460 short(&local_sha),
461 short(&remote_sha)
462 );
463 return Ok(());
464 }
465 crate::reconcile::Reconciliation::Placeholder => {}
466 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467 }
468 }
469 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470 if !out.ok() {
471 bail!(
472 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473 short(&local_sha),
474 short(&remote_sha),
475 out.stderr
476 );
477 }
478 tracing::warn!(
479 "local `{branch}` was stale: fast-forwarded {} -> {}",
480 short(&local_sha),
481 short(&remote_sha)
482 );
483 Ok(())
484}
485
486async fn resolve_base(
487 repo: &Path,
488 base_branch: &str,
489 remote: &str,
490 coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492 let _lease = crate::repo_lock::SeatLease::operation_with(
493 repo,
494 coordination,
495 &crate::repo_lock::WaitPolicy::current(),
496 )
497 .await?;
498 let tracking = format!("{remote}/{base_branch}");
499 let fetched = git::fetch(repo, remote, base_branch).await;
500 if let Ok(out) = &fetched
501 && out.ok()
502 && git::rev_exists(repo, &tracking).await
503 {
504 return Ok(crate::run::FetchedBase {
505 remote: remote.to_owned(),
506 branch: base_branch.to_owned(),
507 commit: git::rev_parse(repo, &tracking).await?,
508 fetched_at: jiff::Timestamp::now(),
509 });
510 }
511 let why = match &fetched {
512 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513 Ok(_) => format!("{remote} has no {base_branch}"),
514 Err(e) => e.to_string(),
515 };
516 bail!(
519 "cannot read {tracking} ({why}); refusing to branch off the local \
520 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521 base / remote in magi.toml. This is launcher preparation: check the \
522 launcher's network, Git access and repository coordination before \
523 retrying"
524 )
525}
526
527enum FixStep {
529 Resource,
530 Recorded { report_lost: bool },
531}
532
533struct FixClaim {
550 path: PathBuf,
551}
552
553impl FixClaim {
554 fn acquire(dir: &Path) -> Result<Self> {
555 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556 let path = dir.join("fix.lock");
557 let retry = crate::lock_file::Retry::new(None);
558 match Self::create(&path, retry) {
559 Ok(claim) => Ok(claim),
560 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561 if Self::reclaim_if_dead(&path) {
562 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563 } else {
564 bail!(
565 "another repair or `magi fix` is already running for this run ({} exists)",
566 path.display()
567 )
568 }
569 }
570 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571 }
572 }
573
574 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575 let mut f = retry.create(path)?;
576 use std::io::Write as _;
577 writeln!(f, "{}", std::process::id())?;
579 Ok(Self {
580 path: path.to_owned(),
581 })
582 }
583
584 fn reclaim_if_dead(path: &Path) -> bool {
588 let dead = std::fs::read_to_string(path)
589 .ok()
590 .and_then(|body| body.trim().parse::<u32>().ok())
591 .is_some_and(|pid| !crate::proc::pid_alive(pid));
592 dead && std::fs::remove_file(path).is_ok()
593 }
594}
595
596impl Drop for FixClaim {
597 fn drop(&mut self) {
598 let _ = std::fs::remove_file(&self.path);
599 }
600}
601
602struct DriverExecution<'a> {
605 runner: &'a mut Runner,
606 driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610 fn drop(&mut self) {
611 self.driver.release_local(&mut self.runner.state);
612 }
613}
614
615impl Runner {
616 pub async fn start(
618 repo: &Path,
619 instruction: String,
620 config: Config,
621 origin: Origin,
622 ) -> Result<Self> {
623 Self::start_naming(repo, instruction, "", config, origin).await
624 }
625
626 pub async fn start_naming(
629 repo: &Path,
630 instruction: String,
631 _also_scan: &str,
632 config: Config,
633 origin: Origin,
634 ) -> Result<Self> {
635 let repo = git::toplevel(repo).await?;
636 let missing = agent::missing_programs(&config.agents);
637 if !missing.is_empty() {
638 bail!(
639 "these agent programs are not on PATH: {}. Fix the roster in \
640 magi.toml or install them.",
641 missing.join(", ")
642 );
643 }
644 let base_branch =
645 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
646 .await?;
647 let fetched_base = resolve_base(
648 &repo,
649 &base_branch,
650 &config.merge.remote,
651 config.coordination.as_ref(),
652 )
653 .await?;
654 let base_commit = fetched_base.commit.clone();
655 let roles = config.resolve_roles()?;
656 let max_parallel = config.graph.max_parallel.max(1);
657 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
658 state.fetched_base = Some(fetched_base);
662 state.origin = Some(origin);
663 state.reserve()?;
664 state.event("start", format!("run {} created", state.id));
665 state.save()?;
666 Ok(Self {
667 state,
668 roles,
669 sem: Arc::new(Semaphore::new(max_parallel)),
670 pause: Pause::new(),
671 interrupt: Pause::new(),
672 })
673 }
674
675 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
689 Self::review_taking_over(repo, branch, config, None, origin).await
690 }
691
692 pub async fn review_taking_over(
699 repo: &Path,
700 branch: &str,
701 config: Config,
702 takeover: Option<crate::handover::Takeover>,
703 origin: Origin,
704 ) -> Result<Self> {
705 let repo = git::toplevel(repo).await?;
706 let missing = agent::missing_programs(&config.agents);
707 if !missing.is_empty() {
708 bail!(
709 "these agent programs are not on PATH: {}. Fix the roster in \
710 magi.toml or install them.",
711 missing.join(", ")
712 );
713 }
714 let base_branch =
715 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
716 .await?;
717 if base_branch == branch {
718 bail!("`{branch}` is the base branch; there is nothing to review against");
719 }
720 let fetched_base = resolve_base(
721 &repo,
722 &base_branch,
723 &config.merge.remote,
724 config.coordination.as_ref(),
725 )
726 .await?;
727 let base_commit = fetched_base.commit.clone();
728
729 let roles = config.resolve_roles()?;
730 let max_parallel = config.graph.max_parallel.max(1);
731 let mut state = RunState::new(
732 repo.clone(),
733 base_branch,
734 base_commit.clone(),
735 String::new(),
736 config,
737 );
738 state.fetched_base = Some(fetched_base);
739 state.origin = Some(origin);
740 state.reserve()?;
741
742 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
749 earlier: Vec::new(),
750 home: crate::run::home(),
751 choice: None,
752 task_instruction: None,
753 });
754 let wait = repository_wait(&state, None);
755 let coordination_config = state.config.coordination.clone();
756 let coordination =
757 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
758 .await?;
759 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
760 if let Some(released) = &released {
761 state.event(
762 "release",
763 format!(
764 "took `{branch}` over from run {}: its worktree was released: {}",
765 crate::run::short_of(&released.old_id),
766 released.audit
767 ),
768 );
769 }
770 state.foreign_checkout = crate::handover::preflight_review(
774 &repo,
775 branch,
776 &takeover.home,
777 &state.config.merge.remote,
778 )
779 .await?;
780 if state.foreign_checkout.is_some() && takeover.choice.is_some() {
781 bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
782 }
783 if let Some(choice) = takeover.choice.as_ref()
784 && let Err(e) =
785 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
786 .await
787 {
788 if let Some(released) = &released {
789 released.restore(&repo, branch).await;
790 }
791 return Err(e.context("applying the owner's answer about the diverged branch"));
792 }
793 for id in takeover.earlier.iter().rev() {
799 if let Ok(old) = RunState::load_under(id, &takeover.home)
800 && let Some(head) = old.pr_head.clone()
801 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
802 {
803 state.pr_head = Some(head);
804 break;
805 }
806 }
807 drop(coordination);
808 let opened = Self::open_review(
809 &repo,
810 branch,
811 state,
812 roles,
813 max_parallel,
814 base_commit,
815 &takeover,
816 )
817 .await;
818 if opened.is_err()
819 && let Some(released) = &released
820 {
821 let _coordination = crate::repo_lock::SeatLease::operation_with(
822 &repo,
823 coordination_config.as_ref(),
824 &wait,
825 )
826 .await?;
827 released.restore(&repo, branch).await;
828 }
829 opened
830 }
831
832 pub async fn recover_pr(
834 repo: &Path,
835 task: &str,
836 selection: crate::pr_recovery::Selection,
837 mut config: Config,
838 ) -> Result<Self> {
839 selection.configure(&mut config)?;
840 selection.check_task(task, &selection.instruction)?;
841 let base_branch =
842 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
843 .await?;
844 let fetched_base = resolve_base(
845 repo,
846 &base_branch,
847 &config.merge.remote,
848 config.coordination.as_ref(),
849 )
850 .await?;
851 let base_commit = fetched_base.commit.clone();
852 selection
853 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
854 .await?;
855 Box::pin(selection.verify_source(repo)).await?;
856 if let Some(proof) = &selection.source_proof {
857 anyhow::ensure!(
858 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
859 "source rebase base is outside the fetched landing base"
860 );
861 }
862 let roles = config.resolve_roles()?;
863 let max_parallel = config.graph.max_parallel.max(1);
864 let mut state = RunState::new(
865 repo.to_path_buf(),
866 base_branch,
867 base_commit.clone(),
868 selection.instruction.clone(),
869 config,
870 );
871 state.fetched_base = Some(fetched_base);
872 state.origin = Some(crate::run::Origin::queue(task));
873 state.pr_head = Some(selection.branch.clone());
874 state.pr_recovery = Some(selection.clone());
875 state.reserve()?;
876 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
877 state.discard_reserved();
879 return Err(e);
880 }
881 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
882 }
883
884 async fn open_selected(
885 repo: &Path,
886 state: RunState,
887 roles: ResolvedRoles,
888 max_parallel: usize,
889 base_commit: String,
890 selection: crate::pr_recovery::Selection,
891 ) -> Result<Self> {
892 let mut state = state;
893 let branch = state.branch_for('A');
894 let wait = repository_wait(&state, None);
895 let prepared = async {
896 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
897 git::git(repo, &["branch", &branch, selection.source()]).await?;
898 drop(coordination);
899 anyhow::Ok(())
900 }
901 .await;
902 if let Err(e) = prepared {
903 state.discard_reserved();
906 return Err(e);
907 }
908 state.event(
909 "recovery",
910 format!(
911 "selected implementation {} for {} (public lease {})",
912 selection.source(),
913 selection.url,
914 selection.head
915 ),
916 );
917 let takeover = crate::handover::Takeover {
918 earlier: Vec::new(),
919 home: crate::run::home(),
920 choice: None,
921 task_instruction: Some(selection.instruction.clone()),
922 };
923 if let Err(e) = state.save() {
924 state.discard_reserved();
925 return Err(e);
926 }
927 let id = state.id.clone();
928 let opened = Self::open_review(
929 repo,
930 &branch,
931 state,
932 roles,
933 max_parallel,
934 base_commit,
935 &takeover,
936 )
937 .await;
938 match opened {
939 Ok(runner) => Ok(runner),
940 Err(e) => {
941 if let Ok(mut partial) = RunState::load(&id) {
942 partial.status = RunStatus::Blocked;
943 partial.driver_exited = true;
944 partial.event(
945 "recovery",
946 format!("startup failed; retained source and candidate {branch}: {e:#}"),
947 );
948 partial.save()?;
949 }
950 Err(crate::pr_recovery::StartupFailed {
951 run: id,
952 cause: format!("{e:#}"),
953 }
954 .into())
955 }
956 }
957 }
958
959 async fn inherit(
964 repo: &Path,
965 branch: &str,
966 base_commit: &str,
967 takeover: &crate::handover::Takeover,
968 ) -> Option<crate::recovery::RecoveryRecord> {
969 use crate::recovery::{Relation, assess};
970 let newest = takeover.earlier.last()?;
971 let mut chosen = None;
972 for id in takeover.earlier.iter().rev() {
973 if let Ok(old) = RunState::load_under(id, &takeover.home)
974 && !old.reviews.is_empty()
975 && (old.winner().is_some_and(|w| w.branch == branch)
976 || old.pr_head.as_deref() == Some(branch))
977 {
978 chosen = Some(old);
979 break;
980 }
981 }
982 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
983 .await
984 .unwrap_or_default();
985 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
986 Some(last) if last.head == tip => Relation::Same,
987 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
988 _ => Relation::Diverged,
989 };
990 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
991 Some(assess(from, chosen.as_ref(), base_commit, relation))
992 }
993
994 async fn open_review(
997 repo: &Path,
998 branch: &str,
999 mut state: RunState,
1000 roles: ResolvedRoles,
1001 max_parallel: usize,
1002 base_commit: String,
1003 takeover: &crate::handover::Takeover,
1004 ) -> Result<Self> {
1005 let wait = repository_wait(&state, None);
1006 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1007 if state.foreign_checkout.is_some() {
1008 if state.pr_head.is_none() {
1010 state.pr_head = Some(branch.to_owned());
1011 }
1012 } else if state.pr_recovery.is_none() {
1013 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1014 }
1015 let start = review_base(
1018 repo,
1019 &state.config.merge.remote,
1020 &state.base_branch,
1021 &base_commit,
1022 branch,
1023 )
1024 .await;
1025 let log = git::log_oneline(repo, &start, branch)
1026 .await
1027 .unwrap_or_default();
1028 let log_text = if log.trim().is_empty() {
1029 "(no commit messages)"
1030 } else {
1031 log.trim()
1032 };
1033 let mut instruction = format!(
1034 "Review the work already on branch `{branch}`. There is no task \
1035 statement: what the change claims to do is whatever its commits \
1036 say.\n\n{log_text}"
1037 );
1038 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1042 if let Some(task) = takeover
1043 .task_instruction
1044 .as_deref()
1045 .filter(|t| !t.trim().is_empty())
1046 && rec.reviewed_head.is_some()
1047 {
1048 instruction = format!(
1049 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1050 task.trim()
1051 );
1052 rec.criteria_inherited = true;
1053 }
1054 state.event("recovery", rec.summary());
1055 state.recovery = Some(rec);
1056 }
1057 if let Some(selected) = &state.pr_recovery {
1058 instruction = selected.instruction.clone();
1059 }
1060 state.task_instruction = takeover.task_instruction.clone();
1061 state.instruction = instruction;
1062 state.reviewed_commits = Some(
1063 git::subjects(repo, &start, branch)
1064 .await
1065 .unwrap_or_default(),
1066 );
1067
1068 for id in &takeover.earlier {
1071 if let Ok(old) = RunState::load_under(id, &takeover.home)
1072 && old.repo == state.repo
1073 && (old.pr_head.as_deref() == Some(branch)
1074 || old.winner().is_some_and(|w| w.branch == branch))
1075 {
1076 for hold in old.branch_owner_holds {
1077 if !state.branch_owner_holds.contains(&hold) {
1078 state.event(
1079 "approval-policy",
1080 format!(
1081 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1082 hold.control.key(),
1083 hold.file,
1084 hold.head
1085 ),
1086 );
1087 state.branch_owner_holds.push(hold);
1088 }
1089 }
1090 }
1091 }
1092 let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1096 let tracking = format!("{}/{branch}", state.config.merge.remote);
1097 let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1100 let mut head = checkout.synced_head.clone();
1101 if matches!(fetched, Ok(out) if out.ok()) {
1102 let remote = git::rev_parse(repo, &tracking).await?;
1103 if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1104 head = remote;
1105 }
1106 }
1107 head
1108 } else {
1109 git::rev_parse(repo, branch).await?
1110 };
1111 state.observe_branch_owner_holds(&reviewed_head).await;
1112
1113 let worktree = state.worktree_root().join("under-review");
1116 if let Some(parent) = worktree.parent() {
1117 tokio::fs::create_dir_all(parent).await.ok();
1118 }
1119 let path = worktree.to_string_lossy().to_string();
1120 let review_branch = if state.foreign_checkout.is_some() {
1121 let shadow = format!("magi/{}/review", state.short());
1122 git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1123 shadow
1124 } else {
1125 branch.to_owned()
1126 };
1127 git::git(repo, &["worktree", "add", &path, &review_branch])
1128 .await
1129 .with_context(|| {
1130 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1131 })?;
1132
1133 let setup = match ensure_setup_config(&mut state, repo).await {
1134 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1135 Err(e) => Err(e),
1136 };
1137 if let Err(e) = setup {
1138 git::worktree_remove_clean(repo, &worktree).await.ok();
1139 return Err(e);
1140 }
1141 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1142 .await
1143 .unwrap_or(0);
1144 if commits == 0 {
1145 git::worktree_remove_clean(repo, &worktree).await.ok();
1146 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1147 }
1148 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1149 .await
1150 .map(|f| f.len())
1151 .unwrap_or(0);
1152 if files == 0
1153 && let (Ok(head_tree), Ok(base_tree)) = (
1154 git::tree_of(&worktree, "HEAD").await,
1155 git::tree_of(&worktree, &base_commit).await,
1156 )
1157 && head_tree == base_tree
1158 {
1159 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1160 git::worktree_remove_clean(repo, &worktree).await.ok();
1161 bail!(
1162 "`{branch}` at {} has a tree identical to base {}; this usually means \
1163 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1164 against `{}/{branch}`) rather than an empty change",
1165 short(&head),
1166 short(&base_commit),
1167 state.config.merge.remote
1168 );
1169 }
1170 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1171 .await
1172 .unwrap_or_default();
1173
1174 state.candidates.push(Candidate {
1175 index: 0,
1176 label: 'A',
1177 agent: EXISTING_BRANCH.to_owned(),
1180 branch: review_branch,
1181 worktree,
1182 summary: String::new(),
1183 stat,
1184 files,
1185 commits,
1186 empty: false,
1187 failed: None,
1188 verified_noop: None,
1189 duration_ms: 0,
1190 folded: false,
1191 });
1192 state.tally = Some(Tally {
1193 first_choice: BTreeMap::from([('A', 0)]),
1194 borda: BTreeMap::new(),
1195 winner: 'A',
1196 rankings: 0,
1197 unanimous_initial: false,
1198 deliberated: false,
1199 changed_votes: 0,
1200 unanimous_final: false,
1201 tie_break: None,
1202 judges: 0,
1206 present: 0,
1207 quorum: 0,
1208 met_quorum: true,
1209 uncontested: Some("review-only run: nothing competed".to_owned()),
1210 });
1211 state.status = RunStatus::Reviewing;
1212 state.prep_complete = Some(true);
1213 state.event(
1214 "start",
1215 format!(
1216 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1217 state.id
1218 ),
1219 );
1220 state.save()?;
1221 Ok(Self {
1222 state,
1223 roles,
1224 sem: Arc::new(Semaphore::new(max_parallel)),
1225 pause: Pause::new(),
1226 interrupt: Pause::new(),
1227 })
1228 }
1229
1230 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1233 Self::authorize_base_recovery_request(home, id, task, false).await
1234 }
1235
1236 pub(crate) async fn authorize_base_recovery_from_resumption(
1238 home: &Path,
1239 id: &str,
1240 task: &str,
1241 ) -> Result<()> {
1242 Self::authorize_base_recovery_request(home, id, task, true).await
1243 }
1244
1245 async fn authorize_base_recovery_request(
1246 home: &Path,
1247 id: &str,
1248 task: &str,
1249 resumption: bool,
1250 ) -> Result<()> {
1251 let _owner = crate::pr_head::owner_lock(home, id)?;
1252 let mut state = RunState::load_under(id, home)?;
1253 crate::pr_head::ensure_owner(&state)?;
1254 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1255 anyhow::ensure!(
1256 queued.runs.last().is_some_and(|run| run == id)
1257 && queued.status != crate::queue::TaskStatus::Done
1258 && !queued.fresh_start
1259 && queued.review_branch.is_none(),
1260 "run task ownership changed"
1261 );
1262 anyhow::ensure!(
1263 state
1264 .origin
1265 .as_ref()
1266 .and_then(|o| o.task.as_deref())
1267 .is_none_or(|owner| owner == task),
1268 "run belongs to another task"
1269 );
1270 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1271 anyhow::ensure!(
1272 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1273 "run is live or its ownership is unknown"
1274 );
1275 let _execution = crate::driver::maintenance(home, id)?;
1278 anyhow::ensure!(
1279 state.status.resumable()
1280 && state.released_to.is_none()
1281 && state.retired_to.is_none()
1282 && state.pr_taken_over_by.is_none(),
1283 "run ownership is final"
1284 );
1285 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1286 anyhow::ensure!(
1287 !state.pr_publication_pending
1288 && !crate::handover::landing_checkpoint(&state)
1289 && state.merge_decision.is_none(),
1290 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1291 );
1292 let winner = state.winner().context("run has no retained winner")?;
1293 anyhow::ensure!(
1294 git::is_clean(&winner.worktree).await?,
1295 "retained worktree has uncommitted changes"
1296 );
1297 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1298 anyhow::ensure!(
1299 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1300 "retained worktree does not match the winner branch"
1301 );
1302 state.record_base_sync_stop(&head);
1303 state.save()?;
1304 if resumption {
1305 anyhow::ensure!(
1306 crate::daemon::compatible_run_instruction(&state, &queued),
1307 "resumption instruction does not match the retained run"
1308 );
1309 let approval = queued
1310 .resumption
1311 .as_ref()
1312 .context("no operator resumption")?;
1313 anyhow::ensure!(
1314 approval.instruction == queued.instruction,
1315 "resumption approval does not match the current task instruction"
1316 );
1317 state.authorize_base_sync_resumption(task, approval)?;
1318 } else {
1319 state.authorize_base_sync(task)?;
1320 }
1321 state.event(
1322 "base-recovery",
1323 "owner authorized one additional base rebase; fresh verification required",
1324 );
1325 state.save()?;
1326 Ok(())
1327 }
1328
1329 pub fn resume(id: &str) -> Result<Self> {
1331 let state = RunState::load(id)?;
1332 crate::pr_head::ensure_owner(&state)?;
1333 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1334 bail!(
1335 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1336 );
1337 }
1338 if let Some(to) = &state.released_to {
1339 bail!(
1340 "run {} cannot be resumed: its worktree was released to run {}",
1341 state.short(),
1342 crate::run::short_of(to)
1343 );
1344 }
1345 let roles = state.config.resolve_roles()?;
1346 if let Some(id) = &state.config.graph.selected_implementer
1347 && state.candidates.iter().any(|c| c.agent != *id)
1348 {
1349 bail!(
1350 "replacement implementer pin does not match the saved candidates; task must stay held"
1351 );
1352 }
1353 let max_parallel = state.config.graph.max_parallel.max(1);
1354 Ok(Self {
1355 state,
1356 roles,
1357 sem: Arc::new(Semaphore::new(max_parallel)),
1358 pause: Pause::new(),
1359 interrupt: Pause::new(),
1360 })
1361 }
1362
1363 pub async fn execute(&mut self) -> Result<()> {
1370 if let Some(selection) = &self.state.pr_recovery
1371 && (selection.source_proof.is_some() || selection.source_run.is_some())
1372 {
1373 Box::pin(selection.verify_source(&self.state.repo)).await?;
1374 }
1375 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1378 || self.state.land_approval.is_some()
1379 || self.state.merge_decision.is_some()
1380 || !self.state.merge_repairs.is_empty()
1381 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1382 {
1383 let claim = FixClaim::acquire(&self.state.dir())?;
1384 let disk = RunState::load(&self.state.id)?;
1385 let checkpoint = |s: &RunState| {
1386 (
1387 s.merge_repairs.len(),
1388 s.merge_repairs
1389 .last()
1390 .map(|r| (r.question.clone(), r.phase)),
1391 s.land_approval
1392 .as_ref()
1393 .map(|a| (a.question.clone(), a.head.clone())),
1394 )
1395 };
1396 if checkpoint(&disk) != checkpoint(&self.state) {
1397 bail!(
1398 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1399 );
1400 }
1401 Some(claim)
1402 } else {
1403 None
1404 };
1405 let mut execution = self.claim_driver().await?;
1406 let runner = &mut *execution.runner;
1407 let wait = runner.repository_wait();
1408 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1411 let result = match result {
1412 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1413 record_repository_block(&mut runner.state, &e);
1414 runner.park_here()?;
1415 Ok(())
1416 }
1417 other => other,
1418 };
1419 execution.driver.finish(&mut runner.state).await?;
1420 let ended = if result.is_err() {
1421 Some({
1422 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1423 if let Err(error) = &result {
1424 notice.message = format!(
1425 "Run {} stopped with an error: {}",
1426 runner.state.id,
1427 error.to_string().lines().next().unwrap_or("unknown error")
1428 );
1429 }
1430 notice
1431 })
1432 } else {
1433 crate::notices::run_ended(&runner.state)
1434 };
1435 if let Some(notice) = ended {
1436 crate::notices::raise(notice);
1437 }
1438 result
1439 }
1440
1441 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1442 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1443 }
1444
1445 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1446 let wait = self.repository_wait();
1447 coordinate_operation(&mut self.state, cwd, &wait).await
1448 }
1449
1450 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1452 let owner = match crate::pr_head::owner_lock_wait(
1453 &crate::run::home(),
1454 &self.state.id,
1455 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1456 )
1457 .await
1458 {
1459 Ok(owner) => owner,
1460 Err(error) => {
1461 let detail = format!("worktree maintenance or ownership update: {error:#}");
1462 self.state.resource_block = Some(crate::run::ResourceBlock {
1466 cause: crate::run::ResourceBlockCause::Other,
1467 node: "start".into(),
1468 kind: "worktree maintenance".into(),
1469 detail: detail.clone(),
1470 at: Timestamp::now(),
1471 });
1472 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1473 }
1474 };
1475 let disk = RunState::load(&self.state.id)?;
1476 crate::pr_head::ensure_owner(&disk)?;
1477 let claimed =
1478 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1479 anyhow::ensure!(
1484 disk.liveness(claimed) != crate::run::Liveness::Live,
1485 "run already has a live driver; wait for its execution to end"
1486 );
1487 if let Some(to) = &disk.released_to {
1488 bail!(
1489 "run {} cannot continue: its worktree was released to run {}",
1490 self.state.short(),
1491 crate::run::short_of(to)
1492 );
1493 }
1494 anyhow::ensure!(
1495 serde_json::to_value(&disk.base_sync_recovery)?
1496 == serde_json::to_value(&self.state.base_sync_recovery)?,
1497 "base recovery changed since this run was loaded; reload the saved run"
1498 );
1499 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1500 let mut execution = DriverExecution {
1501 runner: self,
1502 driver,
1503 };
1504 let _owner = owner;
1507 let state = &mut execution.runner.state;
1508 state.parked = false;
1513 state.resource_wait = None;
1514 state.resource_block = None;
1515 state.clear_active();
1522 let pid = std::process::id();
1537 state.driver_pid = Some(pid);
1538 state.driver_started_at = crate::proc::process_started_at(pid);
1539 state.driver_exited = false;
1540 state.driver_claimed = Some(true);
1541 state.begin_pass();
1542 execution.driver.bind(state);
1543 state.save()?;
1544 execution.driver.record(state)?;
1545 Ok(execution)
1546 }
1547
1548 async fn execute_graph(&mut self) -> Result<()> {
1549 self.state.configure_recovery_resume()?;
1550 self.persist_candidates().await?;
1553 if self.state.base_sync_blocked() {
1556 self.sync_to_base().await?;
1557 if !self.state.base_sync_blocked()
1558 && self.state.status != RunStatus::AlreadyInBase
1559 && !self.resource_halt()?
1560 {
1561 self.finish_after_tally().await?;
1562 }
1563 return Ok(());
1564 }
1565 if self
1566 .state
1567 .merge_decision
1568 .as_ref()
1569 .is_some_and(|decision| !decision.closed)
1570 && self.state.status != RunStatus::Landing
1571 {
1572 if crate::merge_decision::resumable_hold(&self.state) {
1573 self.state.merge = Some(MergeOutcome {
1574 mode: MergeMode::Pr,
1575 ok: false,
1576 detail: self
1577 .state
1578 .merge_decision
1579 .as_ref()
1580 .unwrap()
1581 .target
1582 .pr
1583 .clone(),
1584 empty: false,
1585 });
1586 self.run_land().await?;
1587 self.settle_questions();
1588 return Ok(());
1589 }
1590 bail!(
1591 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1592 );
1593 }
1594 if self.state.config.graph.selected_implementer.is_some()
1595 && self.state.status == RunStatus::Blocked
1596 && self.state.tally.is_none()
1597 && self.state.candidate_preservation.is_empty()
1598 && self.state.interrupted_implementations.is_empty()
1599 {
1600 bail!(
1601 "selected implementation is held; explicitly release the owning task to retry it"
1602 );
1603 }
1604
1605 if self.state.merge_repairs.last().is_some_and(|r| {
1606 matches!(
1607 r.phase,
1608 crate::run::MergeRepairPhase::Requested
1609 | crate::run::MergeRepairPhase::Fixing
1610 | crate::run::MergeRepairPhase::Reviewing
1611 )
1612 }) {
1613 if self.repair_merge_review().await? {
1614 self.run_land().await?;
1615 }
1616 self.settle_questions();
1617 return Ok(());
1618 }
1619 if self.state.pr_publication_pending {
1622 self.merge().await?;
1623 if self.resource_halt()? {
1624 return Ok(());
1625 }
1626 self.state.save()?;
1627 self.settle_questions();
1628 return Ok(());
1629 }
1630 if self.state.status == RunStatus::Stalled {
1643 if self.recover_stall().await? {
1644 self.finish_after_tally().await?;
1645 } else {
1646 self.state.save()?;
1648 }
1649 return Ok(());
1650 }
1651 if self.state.status == RunStatus::Landing
1661 || crate::handover::landing_checkpoint(&self.state)
1662 {
1663 self.state.land_sync_pending = true;
1664 self.state.save()?;
1665 self.run_land().await?;
1666 self.settle_questions();
1671 return Ok(());
1672 }
1673 if self.state.github_text.is_some()
1678 && self.state.merge.is_none()
1679 && self.state.status == RunStatus::Gating
1680 {
1681 self.merge().await?;
1682 if self.resource_halt()? {
1683 return Ok(());
1684 }
1685 self.state.save()?;
1686 return Ok(());
1687 }
1688 if let Err(e) = self.prep().await {
1689 if e.is::<crate::repo_lock::ResourceBlocked>() {
1690 return Err(e);
1691 }
1692 self.state.status = RunStatus::Blocked;
1693 self.state
1694 .prep_failure
1695 .get_or_insert_with(|| format!("{e:#}"));
1696 self.state
1697 .event("prep", format!("preparation stopped: {e:#}"));
1698 self.state.save()?;
1699 return Err(e);
1700 }
1701 if self.park_here()? {
1702 return Ok(());
1703 }
1704 self.advise().await?;
1705 if self.park_here()? {
1706 return Ok(());
1707 }
1708 self.implement().await?;
1709 if self.park_here()? {
1710 return Ok(());
1711 }
1712 if self.state.status == RunStatus::VerifiedNoop {
1716 return Ok(());
1717 }
1718 self.judge().await?;
1719 if self.park_here()? {
1720 return Ok(());
1721 }
1722 self.deliberate().await?;
1723 if self.park_here()? {
1724 return Ok(());
1725 }
1726 self.vote().await?;
1727 if self.park_here()? {
1728 return Ok(());
1729 }
1730 self.tally()?;
1731 if self.state.status == RunStatus::Stalled {
1736 self.state.save()?;
1740 return Ok(());
1741 }
1742 self.finish_after_tally().await?;
1743 Ok(())
1744 }
1745
1746 fn stop_for_resource(&mut self) -> Result<()> {
1751 if self.pause.parked() || self.interrupt.parked() {
1754 self.park_here()?;
1755 return Ok(());
1756 }
1757 if self.state.resource_block.is_some() {
1758 self.state.status = RunStatus::Blocked;
1759 }
1760 self.state.save()
1761 }
1762
1763 fn resource_halt(&mut self) -> Result<bool> {
1765 if self.state.resource_block.is_none() {
1766 return Ok(false);
1767 }
1768 self.stop_for_resource()?;
1769 Ok(true)
1770 }
1771
1772 fn park_here(&mut self) -> Result<bool> {
1779 let parking = self.pause.parked() || self.interrupt.parked();
1780 if !parking && self.state.resource_block.is_some() {
1781 self.stop_for_resource()?;
1782 return Ok(true);
1783 }
1784 if !self.pause.parked() && !self.interrupt.parked() {
1789 return Ok(false);
1790 }
1791 if self.state.parked {
1792 return Ok(true);
1793 }
1794 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1795 if let Some(block) = &mut self.state.resource_block
1796 && matches!(
1797 block.cause,
1798 crate::run::ResourceBlockCause::Cancelled { .. }
1799 )
1800 {
1801 block.cause = crate::run::ResourceBlockCause::Cancelled {
1802 park: cause.clone(),
1803 };
1804 }
1805 let why = match cause.as_ref().map(ParkCause::label) {
1806 Some(reason) => format!(
1807 "parked after `{}` ({reason}) — resume to carry on from here",
1808 self.state.status.as_str()
1809 ),
1810 None => format!(
1811 "parked after `{}` — resume to carry on from here",
1812 self.state.status.as_str()
1813 ),
1814 };
1815 self.state.event("park", why);
1816 self.state
1817 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1818 self.state.parked = true;
1819 self.state.save()?;
1820 Ok(true)
1821 }
1822
1823 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1825 if self.state.prep_failure.is_none() {
1826 return Ok(());
1827 }
1828 let failed_at = self
1829 .state
1830 .events
1831 .iter()
1832 .rev()
1833 .find(|event| event.node == "prep")
1834 .map_or(self.state.updated_at, |event| event.at);
1835 anyhow::ensure!(
1836 approval_at > failed_at,
1837 "preparation retry requires a new explicit release after the failure"
1838 );
1839 self.state.prep_failure = None;
1840 self.state.prep_complete = Some(false);
1841 self.state
1842 .event("prep-retry", "operator approved retry of preparation");
1843 Ok(())
1844 }
1845
1846 pub fn on_pause(&mut self, pause: Pause) {
1848 self.pause = pause;
1849 }
1850
1851 pub fn watch_interrupt(&mut self, pause: Pause) {
1857 self.interrupt = pause;
1858 }
1859
1860 fn settle_questions(&mut self) {
1880 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1881 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1882 }
1883 }
1884
1885 async fn finish_after_tally(&mut self) -> Result<()> {
1888 self.graft().await?;
1891 if self.resource_halt()? {
1892 return Ok(());
1893 }
1894 self.fold_losers().await?;
1895 self.sync_to_base().await?;
1900 if self.hold_base_sync().await? || self.resource_halt()? {
1901 return Ok(());
1902 }
1903 if self.state.status == RunStatus::AlreadyInBase {
1904 return Ok(());
1905 }
1906 self.review_loop().await?;
1907 if self.resource_halt()? {
1908 return Ok(());
1909 }
1910 self.sync_to_base().await?;
1911 if self.hold_base_sync().await? || self.resource_halt()? {
1912 return Ok(());
1913 }
1914 if self.state.status == RunStatus::AlreadyInBase {
1915 return Ok(());
1916 }
1917 self.gate().await?;
1918 self.merge().await?;
1919 if self.resource_halt()? {
1920 return Ok(());
1921 }
1922 self.state.save()?;
1923 Ok(())
1924 }
1925
1926 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1933 let result = match ensure_setup_config(&mut self.state, repo).await {
1934 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1935 Err(e) => Err(e),
1936 };
1937 if let Err(e) = result {
1938 git::worktree_remove_clean(repo, wt).await.ok();
1939 if in_prep {
1940 self.rollback_prep(repo).await;
1941 }
1942 self.state.status = RunStatus::Blocked;
1943 self.state.event(
1944 "setup",
1945 format!("worktree setup failed in {}: {e:#}", wt.display()),
1946 );
1947 self.state.save()?;
1948 return Err(e);
1949 }
1950 Ok(())
1951 }
1952
1953 async fn rollback_prep(&mut self, repo: &Path) {
1955 let root = self.state.worktree_root();
1956 for c in std::mem::take(&mut self.state.candidates) {
1957 if matches!(
1958 git::worktree_remove_clean(repo, &c.worktree).await,
1959 Ok(true)
1960 ) {
1961 git::branch_delete(repo, &c.branch).await.ok();
1962 } else {
1963 self.state.candidates.push(c);
1964 }
1965 }
1966 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1967 for j in 1..=seats {
1968 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1969 .await
1970 .ok();
1971 }
1972 for k in 1..=self.state.config.graph.advisors {
1973 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1974 .await
1975 .ok();
1976 }
1977 }
1978
1979 async fn prep(&mut self) -> Result<()> {
1980 if let Some(reason) = &self.state.prep_failure {
1981 bail!(
1982 "candidate preparation previously failed: {reason}; correct the source before retrying"
1983 );
1984 }
1985 if self
1986 .state
1987 .prep_complete
1988 .unwrap_or(!self.state.candidates.is_empty())
1989 {
1990 return Ok(());
1991 }
1992 let repo = self.state.repo.clone();
1993 let _coordination = self.coordinate(&repo).await?;
1996 let base = self.state.base_commit.clone();
1997 let plan = self.state.sources.plan(&repo, &base).await?;
1998 self.rollback_prep(&repo).await;
1999 if !self.state.candidates.is_empty() {
2000 bail!(
2001 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2002 );
2003 }
2004 self.state.prep_complete = Some(false);
2005 self.state.status = RunStatus::Prep;
2006 self.state.save()?;
2007 let repo = self.state.repo.clone();
2008 let start = plan.start.clone().unwrap_or_else(|| base.clone());
2009 let root = self.state.worktree_root();
2010 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2011
2012 let hooks_dir = self.state.dir().join("hooks");
2015 if self.state.config.blind.commit_msg_hook {
2016 std::fs::create_dir_all(&hooks_dir)
2017 .with_context(|| format!("create {}", hooks_dir.display()))?;
2018 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2019 let path = hooks_dir.join("commit-msg");
2020 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2021 make_executable(&path)?;
2022 git::acquire_worktree_config(&repo).await?;
2030 self.state.enabled_worktree_config = true;
2031 }
2032
2033 ensure_setup_config(&mut self.state, &repo).await?;
2036
2037 for (index, (spec, label)) in self
2038 .roles
2039 .implementers
2040 .clone()
2041 .into_iter()
2042 .zip(labels)
2043 .enumerate()
2044 {
2045 let branch = self.state.branch_for(label);
2046 let worktree = root.join(format!("cand-{label}"));
2047 self.state.event(
2051 "prep-intent",
2052 serde_json::json!({
2053 "branch": branch,
2054 "worktree": worktree,
2055 "start": start,
2056 "picks": plan.picks,
2057 })
2058 .to_string(),
2059 );
2060 self.state.save()?;
2061 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2062 if self.state.config.blind.commit_msg_hook {
2063 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2064 }
2065 git::local_exclude(&worktree, "/.magi/").await?;
2066 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2067 git::worktree_remove_clean(&repo, &worktree).await.ok();
2070 git::branch_delete(&repo, &branch).await.ok();
2071 self.rollback_prep(&repo).await;
2072 self.state.status = RunStatus::Blocked;
2073 self.state
2074 .event("prep", format!("worktree setup failed: {e:#}"));
2075 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2076 self.state.save()?;
2077 return Err(e);
2078 }
2079 self.state.candidates.push(Candidate {
2080 index,
2081 label,
2082 agent: spec.id.clone(),
2083 branch,
2084 worktree,
2085 summary: String::new(),
2086 stat: String::new(),
2087 files: 0,
2088 commits: 0,
2089 empty: false,
2090 failed: None,
2091 verified_noop: None,
2092 duration_ms: 0,
2093 folded: false,
2094 });
2095 self.state.save()?;
2096 for pick in &plan.picks {
2097 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2098 {
2099 self.state.status = RunStatus::Blocked;
2100 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2101 self.state
2102 .event("prep", format!("cannot apply referenced commit: {e}"));
2103 self.state.save()?;
2104 return Err(e);
2105 }
2106 }
2107 }
2108
2109 for j in 1..=self.roles.judges.len() {
2110 let wt = root.join(format!("judge-{j}"));
2111 if !wt.exists() {
2112 git::worktree_add_detached(&repo, &wt, &base).await?;
2113 self.setup_seat_worktree(&repo, &wt, true).await?;
2114 }
2115 }
2116
2117 if self.state.config.graph.advise {
2125 for k in 1..=self.state.config.graph.advisors {
2126 let wt = root.join(format!("advisor-{k}"));
2127 if !wt.exists() {
2128 git::worktree_add_detached(&repo, &wt, &base).await?;
2129 self.setup_seat_worktree(&repo, &wt, true).await?;
2130 }
2131 }
2132 }
2133
2134 let authors: Vec<&str> = self
2139 .roles
2140 .implementers
2141 .iter()
2142 .map(|a| a.id.as_str())
2143 .collect();
2144 let overlap: Vec<String> = self
2145 .roles
2146 .judges
2147 .iter()
2148 .enumerate()
2149 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2150 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2151 .collect();
2152 if !overlap.is_empty() {
2153 let note = format!(
2154 "{} also authored a candidate; blind, but the panel is less \
2155 independent than {} distinct agents would be",
2156 overlap.join(", "),
2157 self.roles.judges.len()
2158 );
2159 self.state.event("prep", note);
2160 }
2161
2162 self.state.event(
2163 "prep",
2164 format!(
2165 "{} candidates, {} judges, base {} ({})",
2166 self.state.candidates.len(),
2167 self.roles.judges.len(),
2168 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2169 self.state.base_branch
2170 ),
2171 );
2172 self.state.prep_complete = Some(true);
2173 self.state.status = RunStatus::Implementing;
2174 self.state.save()?;
2175 Ok(())
2176 }
2177
2178 async fn advise(&mut self) -> Result<()> {
2211 let implement_untouched = self
2212 .state
2213 .candidates
2214 .iter()
2215 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2216 if !self.state.config.graph.advise || self.state.advise_attempted {
2217 return Ok(());
2218 }
2219 if !implement_untouched {
2220 self.state.event(
2221 "advise",
2222 "skipping the design-deliberation stage: at least one \
2223 candidate already shows implementation progress, so this \
2224 run is past the point the stage exists to run before"
2225 .to_owned(),
2226 );
2227 self.state.advise_attempted = true;
2228 self.state.save()?;
2229 return Ok(());
2230 }
2231 let run_id = self.state.id.clone();
2232 let prompts = self.state.config.prompts.clone();
2233 let instruction = self.state.instruction.clone();
2234 let language = self.state.config.graph.language.clone();
2235 let root = self.state.worktree_root();
2236 let n = self.state.config.graph.advisors;
2237 let where_recorded = self.state.dir().join("run.json");
2238
2239 let seats = match self.state.config.advisors() {
2240 Ok(seats) if !seats.is_empty() => seats,
2241 Ok(_) => {
2242 self.state.event(
2243 "advise",
2244 format!(
2245 "[graph] advisors is 0; skipping the design-deliberation \
2246 stage and continuing without a synthesis brief (see {})",
2247 where_recorded.display()
2248 ),
2249 );
2250 self.state.advise_attempted = true;
2251 self.state.save()?;
2252 return Ok(());
2253 }
2254 Err(e) => {
2255 self.state.event(
2256 "advise",
2257 format!(
2258 "could not resolve advisor seats ({e:#}); continuing \
2259 without a design-deliberation brief (see {})",
2260 where_recorded.display()
2261 ),
2262 );
2263 self.state.advise_attempted = true;
2264 self.state.save()?;
2265 return Ok(());
2266 }
2267 };
2268
2269 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2270 let artifacts = agent::artifacts_dir(&self.state.dir());
2271 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2272
2273 let mut jobs = Vec::new();
2274 for (i, spec) in seats.iter().cloned().enumerate() {
2275 let seat_key = format!("advisor-{}", i + 1);
2276 let seat = self.seat(&seat_key, &spec.id);
2277 jobs.push(SeatJob {
2278 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2279 spec,
2280 seat,
2281 cwd: worktrees[i % worktrees.len()].clone(),
2282 timeout,
2283 allow_write: false,
2284 sessions: false,
2285 artifacts: artifacts.clone(),
2286 stem: seat_key,
2287 handover: None,
2288 });
2289 }
2290
2291 self.state.event(
2292 "advise",
2293 format!(
2294 "{} advisor seat(s) sketching a design in parallel",
2295 jobs.len()
2296 ),
2297 );
2298 let mut quota_losses = Vec::new();
2299 let cache = self.state.config.cache_dir();
2300 let ctx = WaveCtx {
2301 carry_seats: false,
2302 pauses: Some((&self.pause, &self.interrupt)),
2303 run: &run_id,
2304 node: "advise",
2305 prompts: &prompts,
2306 cache: cache.as_deref(),
2307 round: None,
2308 };
2309 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2310 let results = ask_json_wave::<Proposal>(
2311 jobs,
2312 Arc::clone(&self.sem),
2313 self.state.config.graph.retries,
2314 &advisor_roster,
2315 &ctx,
2316 &mut quota_losses,
2317 &mut self.state,
2318 &|p: &Proposal| p.validate(),
2319 )
2320 .await;
2321 if self.state.resource_block.is_some() {
2322 return self.stop_for_resource();
2323 }
2324 self.state.quota.extend(quota_losses);
2325
2326 let mut records = Vec::with_capacity(results.len());
2327 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2328 let agent_id = seat.agent.clone();
2329 self.state.seats.insert(seat.key.clone(), seat);
2330 match res {
2331 Ok((proposal, out)) => {
2332 self.state
2333 .event("advise", format!("advisor-{} proposed a design", i + 1));
2334 records.push(advise::AdvisorRecord::proposed(
2335 i + 1,
2336 agent_id,
2337 proposal,
2338 out.duration_ms,
2339 ));
2340 }
2341 Err(e) => {
2342 self.state.event(
2343 "advise",
2344 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2345 );
2346 records.push(advise::AdvisorRecord::failed(
2347 i + 1,
2348 agent_id,
2349 e.to_string(),
2350 ));
2351 }
2352 }
2353 }
2354
2355 let mut advice = advise::Advice {
2356 records,
2357 synthesis: None,
2358 };
2359 if advice.proposals().is_empty() {
2360 self.state.event(
2361 "advise",
2362 "no advisor produced a usable proposal; continuing without a \
2363 synthesis brief"
2364 .to_owned(),
2365 );
2366 } else {
2367 match self
2368 .synthesize_brief(
2369 &advice,
2370 &instruction,
2371 &language,
2372 &worktrees[0],
2373 &artifacts,
2374 &run_id,
2375 &prompts,
2376 cache.as_deref(),
2377 )
2378 .await
2379 {
2380 Ok(Some(text)) => {
2381 self.state.event(
2382 "advise",
2383 "synthesized a design brief for the implementer".to_owned(),
2384 );
2385 advice.synthesis = Some(text);
2386 }
2387 Ok(None) => {
2388 self.state.event(
2389 "advise",
2390 "the synthesis seat produced nothing usable; continuing \
2391 without a design brief"
2392 .to_owned(),
2393 );
2394 }
2395 Err(e) => {
2396 if e.is::<crate::repo_lock::ResourceBlocked>() {
2397 return Err(e);
2398 }
2399 self.state.event(
2400 "advise",
2401 format!("could not synthesize a design brief: {e:#}"),
2402 );
2403 }
2404 }
2405 }
2406 advise::apply_reflection(&mut advice);
2407
2408 self.state.advice = Some(advice);
2409 self.state.advise_attempted = true;
2410 self.state.save()?;
2411 Ok(())
2412 }
2413
2414 #[allow(clippy::too_many_arguments)]
2426 async fn synthesize_brief(
2427 &mut self,
2428 advice: &advise::Advice,
2429 instruction: &str,
2430 language: &str,
2431 cwd: &Path,
2432 artifacts: &Path,
2433 run_id: &str,
2434 prompts: &Prompts,
2435 cache: Option<&Path>,
2436 ) -> Result<Option<String>> {
2437 let chain = agent::pick_chain(
2438 &self.state.config.agents,
2439 self.state.config.roles.synthesizer.as_ref(),
2440 &agent::installed,
2441 "synthesizer",
2442 )?;
2443 let proposals = advice.proposals();
2444 let mut prompt = prompt::with_overlay(
2445 prompt::synthesize_brief(instruction, &proposals, language),
2446 prompts.overlay("advise"),
2447 );
2448 if cache.is_some() {
2449 prompt.push('\n');
2454 prompt.push_str(&prompt::build_cache_note("advise", false));
2455 }
2456 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2457 let mut last = None;
2462 for (n, spec) in chain.iter().enumerate() {
2463 if n > 0 {
2464 self.state
2465 .event("advise", format!("synthesis falling back to {}", spec.id));
2466 }
2467 let mut seat = self.seat("advise-synthesis", &spec.id);
2468 let wait = self.repository_wait();
2469 let outcome = wait
2470 .scope(agent::invoke(
2471 spec,
2472 &mut seat,
2473 &Invocation {
2474 coordination: self.state.config.coordination.as_ref(),
2475 fetched_base: None,
2476 cwd,
2477 prompt: &prompt,
2478 timeout,
2479 allow_write: false,
2480 sessions: false,
2481 artifacts,
2482 stem: &if n == 0 {
2483 "advise-synthesis".to_owned()
2484 } else {
2485 format!("advise-synthesis-{}", spec.id)
2486 },
2487 run: run_id,
2488 node: "advise",
2489 cache_dir: None,
2490 attachments: &[],
2491 writable: &[],
2492 },
2493 ))
2494 .await;
2495 if outcome.is_ok() {
2496 self.state.seats.insert(seat.key.clone(), seat);
2497 }
2498 let advance = agent::chain_advances(&outcome);
2499 last = Some(outcome);
2500 if !advance {
2501 break;
2502 }
2503 }
2504 let out = last.expect("a chain holds at least one agent")?;
2507 if !out.usable() {
2508 return Ok(None);
2509 }
2510 let text =
2511 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2512 Ok((!text.trim().is_empty()).then_some(text))
2513 }
2514
2515 async fn implement(&mut self) -> Result<()> {
2518 self.persist_candidates().await?;
2519 let run_id = self.state.id.clone();
2524 let prompts = self.state.config.prompts.clone();
2525 let todo: Vec<usize> = self
2526 .state
2527 .candidates
2528 .iter()
2529 .enumerate()
2530 .filter(|(_, c)| {
2531 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2532 && c.failed.is_none()
2533 && !c.empty
2534 })
2535 .map(|(i, _)| i)
2536 .collect();
2537 if todo.is_empty() {
2538 return self.after_implement();
2539 }
2540 self.state.status = RunStatus::Implementing;
2541
2542 let language = self.state.config.graph.language.clone();
2543 let github_language = self.state.config.graph.github_language.clone();
2544 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2545 let sessions = self.state.config.graph.sessions;
2546 let artifacts = agent::artifacts_dir(&self.state.dir());
2547 let brief = self
2551 .state
2552 .advice
2553 .as_ref()
2554 .and_then(|a| a.synthesis.as_deref())
2555 .map(str::to_owned);
2556 let attachments = self.state.attachments.clone();
2557
2558 let mut jobs = Vec::new();
2559 for &i in &todo {
2560 let (index, label, worktree) = {
2561 let c = &self.state.candidates[i];
2562 (c.index, c.label, c.worktree.clone())
2563 };
2564 let spec = self.roles.implementers[index].clone();
2565 let seat_key = format!("impl-{label}");
2566 let seat = self.seat(&seat_key, &spec.id);
2567 let instruction = seeded_instruction(&self.state);
2568 jobs.push(SeatJob {
2569 spec,
2570 seat,
2571 prompt: prompt::implement_in(
2572 &instruction,
2573 &worktree.to_string_lossy(),
2574 &language,
2575 &github_language,
2576 brief.as_deref(),
2577 &attachments,
2578 ),
2579 cwd: worktree,
2580 timeout,
2581 allow_write: true,
2582 sessions,
2583 artifacts: artifacts.clone(),
2584 stem: format!("impl-{label}"),
2585 handover: None,
2586 });
2587 }
2588
2589 self.state.event(
2590 "implement",
2591 format!("{} candidates in parallel", jobs.len()),
2592 );
2593 let mut sent = jobs.clone();
2599 let cache = self.state.config.cache_dir();
2600 let ctx = WaveCtx {
2601 carry_seats: false,
2602 pauses: Some((&self.pause, &self.interrupt)),
2603 run: &run_id,
2604 node: "implement",
2605 prompts: &prompts,
2606 cache: cache.as_deref(),
2607 round: None,
2608 };
2609 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2610 if !wave_blocked(&results) {
2611 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2612 .await;
2613 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2614 .await?;
2615 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2616 .await;
2617 }
2618 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2619
2620 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2621 && results
2622 .iter()
2623 .any(|(_, _, out)| FailClass::of(out).is_some());
2624 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2625 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2626 if self
2630 .state
2631 .candidate_preservation
2632 .contains_key(&self.state.candidates[i].label)
2633 {
2634 self.state.seats.insert(seat.key.clone(), seat);
2635 }
2636 continue;
2637 }
2638 let seat_key = seat.key.clone();
2639 let agent = seat.agent.clone();
2648 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2649 self.state.seats.insert(seat.key.clone(), seat);
2650 let label = self.state.candidates[i].label;
2651 self.state.interrupted_implementations.remove(&label);
2652
2653 let (summary, duration, failed, verified_claim) = match out {
2654 AgentOutcome::Ok(o) => {
2655 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2656 let failed = (!o.usable()).then(|| {
2657 if o.timed_out {
2658 "agent timed out".to_owned()
2659 } else {
2660 format!(
2661 "agent exited with {}",
2662 crate::notices::exit_status(o.exit_code)
2663 )
2664 }
2665 });
2666 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2667 (text, o.duration_ms, failed, verified_claim)
2668 }
2669 AgentOutcome::Dropped(o) => {
2675 let why = o
2676 .dropped
2677 .as_ref()
2678 .map(|d| d.why.as_str())
2679 .unwrap_or("the CLI ended the stream without delivering its answer");
2680 (
2681 String::new(),
2682 o.duration_ms,
2683 Some(format!("the CLI dropped the stream ({why})")),
2684 None,
2685 )
2686 }
2687 AgentOutcome::Quota(o) => {
2688 self.state.quota.push(QuotaLoss {
2689 seat: seat_key,
2690 node: "implement".to_owned(),
2691 at: Timestamp::now(),
2692 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2693 });
2694 (
2695 String::new(),
2696 o.duration_ms,
2697 Some("rate limited (quota); produced no change".to_owned()),
2698 None,
2699 )
2700 }
2701 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2702 (String::new(), 0, Some(e), None)
2703 }
2704 };
2705
2706 let c = &mut self.state.candidates[i];
2709 if !exhausted_the_fallback_chain {
2710 c.agent = agent;
2711 }
2712 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2713 c.duration_ms = duration;
2714 c.failed = failed;
2715 c.verified_noop = verified_claim;
2716 self.state
2717 .candidate_preservation
2718 .insert(label, "awaiting persistence".to_owned());
2719 }
2720 self.state.save()?;
2721 self.persist_candidates().await?;
2722
2723 if resource_blocked {
2724 return self.stop_for_resource();
2725 }
2726 if selected_failed {
2727 self.state.status = RunStatus::Blocked;
2728 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2729 self.state.save()?;
2730 bail!(
2731 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2732 );
2733 }
2734 self.after_implement()
2735 }
2736
2737 async fn persist_candidates(&mut self) -> Result<()> {
2739 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2740 for label in labels {
2741 let i = self
2742 .state
2743 .candidates
2744 .iter()
2745 .position(|c| c.label == label)
2746 .context("pending candidate is missing")?;
2747 let c = self.state.candidates[i].clone();
2748 let interrupted = self.state.interrupted_implementations.contains(&label);
2749 let result = async {
2750 let wait = crate::repo_lock::WaitPolicy::new(
2751 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2752 || false,
2753 );
2754 let _coordination =
2755 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2756 if let Err(e) = git::snapshot_uncommitted(
2759 &c.worktree,
2760 &self
2761 .state
2762 .dir()
2763 .join("artifacts")
2764 .join(format!("candidate-{label}-recovery")),
2765 )
2766 .await
2767 {
2768 self.state.event(
2769 "preserve",
2770 format!("candidate {label} snapshot incomplete: {e:#}"),
2771 );
2772 }
2773 let rescued = git::rescue_commit(
2774 &c.worktree,
2775 &format!("magi: candidate {label} (uncommitted work)"),
2776 )
2777 .await?;
2778 self.state.note_withheld("implement", &rescued.withheld);
2779 let base = &self.state.base_commit;
2780 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2781 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2782 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2783 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2784 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2785 if c.empty && commits > 0 && !patch.trim().is_empty()
2786 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2787 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2788 || self.state.tally.is_some())
2789 {
2790 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2794 }
2795 let c = &mut self.state.candidates[i];
2796 c.commits = commits;
2797 c.stat = stat;
2798 c.files = files;
2799 let no_changes = commits == 0 || patch.trim().is_empty();
2800 c.empty = no_changes && !interrupted;
2801 if interrupted && no_changes {
2802 c.failed = None;
2803 c.verified_noop = None;
2804 }
2805 if !c.empty {
2806 c.failed = None;
2807 c.verified_noop = None;
2808 }
2809 self.state.event(
2810 "preserve",
2811 format!("candidate {label}: {files} files, {commits} commits"),
2812 );
2813 Ok::<_, anyhow::Error>(())
2814 }
2815 .await;
2816 match result {
2817 Ok(()) => {
2818 self.state.candidate_preservation.remove(&label);
2819 if !interrupted || self.state.candidates[i].files > 0 {
2820 self.state.interrupted_implementations.remove(&label);
2821 }
2822 }
2823 Err(e) => {
2824 let reason = format!("candidate {label} preservation failed: {e:#}");
2825 self.state.event("preserve", &reason);
2826 self.state.candidate_preservation.insert(label, reason);
2827 }
2828 }
2829 if self.state.candidate_preservation.is_empty()
2830 && self.state.status == RunStatus::Blocked
2831 && self.state.candidates.iter().all(|c| c.failed.is_none())
2832 {
2833 self.state.status = RunStatus::Implementing;
2837 self.state.event(
2838 "preserve",
2839 "candidate persistence recovered; implementation can continue",
2840 );
2841 }
2842 self.state.save()?;
2843 }
2844 if !self.state.candidate_preservation.is_empty() {
2845 self.state.status = RunStatus::Blocked;
2846 self.state.save()?;
2847 bail!(
2848 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2849 );
2850 }
2851 Ok(())
2852 }
2853
2854 async fn resume_undelivered(
2882 &mut self,
2883 results: &mut [(usize, SeatState, AgentOutcome)],
2884 sent: &[SeatJob],
2885 prompts: &Prompts,
2886 run_id: &str,
2887 ) {
2888 for (wi, seat, out) in results.iter_mut() {
2889 let Some(dropped) = (match &*out {
2890 AgentOutcome::Dropped(o) => o.dropped.clone(),
2891 _ => None,
2892 }) else {
2893 continue;
2894 };
2895 let Some(job) = sent.get(*wi) else { continue };
2896 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2898 self.state.event(
2899 "implement",
2900 format!(
2901 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2902 work is in the tree",
2903 seat.key, dropped.output_tokens, dropped.why
2904 ),
2905 );
2906 continue;
2907 }
2908 if !has_context(&job.spec, seat, job.sessions) {
2916 self.state.event(
2917 "implement",
2918 format!(
2919 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2920 is no session left to resume",
2921 seat.key, dropped.output_tokens, dropped.why
2922 ),
2923 );
2924 continue;
2925 }
2926 self.state.event(
2927 "implement",
2928 format!(
2929 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2930 conversation",
2931 seat.key, dropped.output_tokens, dropped.why
2932 ),
2933 );
2934 let mut retry = job.clone();
2935 retry.seat = seat.clone();
2936 retry.prompt = prompt::resume_after_drop(&dropped.why);
2937 retry.timeout = retry_budget(job.timeout, true);
2938 retry.stem = format!("{}-resume", job.stem);
2939 let cache = self.state.config.cache_dir();
2940 let ctx = WaveCtx {
2941 carry_seats: false,
2942 pauses: Some((&self.pause, &self.interrupt)),
2943 run: run_id,
2944 node: "implement",
2945 prompts,
2946 cache: cache.as_deref(),
2947 round: None,
2948 };
2949 let (resumed_seat, resumed) =
2950 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2951 *seat = resumed_seat;
2952 *out = resumed;
2953 }
2954 }
2955
2956 async fn resume_seat_handovers(
3021 &mut self,
3022 results: &mut [(usize, SeatState, AgentOutcome)],
3023 sent: &mut [SeatJob],
3024 prompts: &Prompts,
3025 run_id: &str,
3026 ) -> Result<()> {
3027 let instruction = seeded_instruction(&self.state);
3028 let language = self.state.config.graph.language.clone();
3029 let github_language = self.state.config.graph.github_language.clone();
3030 let brief = self
3031 .state
3032 .advice
3033 .as_ref()
3034 .and_then(|a| a.synthesis.as_deref())
3035 .map(str::to_owned);
3036 let attachments = self.state.attachments.clone();
3037 for (wi, seat, out) in results.iter_mut() {
3038 let others: BTreeSet<String> = sent
3041 .iter()
3042 .enumerate()
3043 .filter(|(j, _)| j != wi)
3044 .map(|(_, j)| j.spec.id.clone())
3045 .collect();
3046 let Some(job) = sent.get_mut(*wi) else {
3047 continue;
3048 };
3049 let start = self
3054 .roles
3055 .implementer_roster
3056 .iter()
3057 .position(|s| s.id == job.spec.id)
3058 .unwrap_or(0);
3059 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3060 let mut fallback_attempt = 0usize;
3061 let mut prev: Option<FailClass> = None;
3062 while let Some(cur) = FailClass::of(&*out) {
3063 if !should_hand_over(prev.as_ref(), &cur) {
3064 break;
3065 }
3066 let Some(next) =
3067 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3068 .cloned()
3069 else {
3070 break;
3071 };
3072 tried.insert(next.id.clone());
3073 fallback_attempt += 1;
3074
3075 let coordination = self.coordinate(&job.cwd).await?;
3076 if let Ok(r) = git::rescue_commit(
3077 &job.cwd,
3078 &format!(
3079 "magi: candidate {} (uncommitted work before {} fallback)",
3080 seat.key,
3081 if cur == FailClass::Quota {
3082 "quota"
3083 } else {
3084 "handover"
3085 }
3086 ),
3087 )
3088 .await
3089 {
3090 self.state.note_withheld("implement", &r.withheld);
3091 }
3092 drop(coordination);
3093
3094 record_handover(
3095 &mut self.state,
3096 "implement",
3097 &seat.key,
3098 &seat.agent,
3099 &next.id,
3100 &cur,
3101 &fail_reason(&*out),
3102 );
3103 prev = Some(cur.clone());
3104
3105 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3106 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3107 job.spec = next.clone();
3115 let mut retry = job.clone();
3116 retry.seat = new_seat;
3117 retry.prompt = prompt::implement_in(
3118 &instruction,
3119 &job.cwd.to_string_lossy(),
3120 &language,
3121 &github_language,
3122 brief.as_deref(),
3123 &attachments,
3124 );
3125 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3126 let cache = self.state.config.cache_dir();
3127 let ctx = WaveCtx {
3128 carry_seats: false,
3129 pauses: Some((&self.pause, &self.interrupt)),
3130 run: run_id,
3131 node: "implement",
3132 prompts,
3133 cache: cache.as_deref(),
3134 round: None,
3135 };
3136 let (fallback_seat, fallback_out) = run_one(
3137 retry,
3138 Arc::clone(&self.sem),
3139 &ctx,
3140 &mut self.state,
3141 fallback_attempt,
3142 )
3143 .await;
3144 *seat = fallback_seat;
3145 *out = fallback_out;
3146 }
3147 }
3148 Ok(())
3149 }
3150
3151 async fn resume_unconfirmed_commands(
3175 &mut self,
3176 results: &mut [(usize, SeatState, AgentOutcome)],
3177 sent: &[SeatJob],
3178 prompts: &Prompts,
3179 run_id: &str,
3180 ) {
3181 for (wi, seat, out) in results.iter_mut() {
3182 let AgentOutcome::Ok(o) = &*out else {
3183 continue;
3184 };
3185 if !has_unconfirmed_command(&o.commands) {
3186 continue;
3187 }
3188 let Some(job) = sent.get(*wi) else { continue };
3189 if !has_context(&job.spec, seat, job.sessions) {
3190 self.state.event(
3191 "implement",
3192 format!(
3193 "{}: the reply named a command whose own CLI never confirmed the exit \
3194 status of, but there is no session left to resume",
3195 seat.key
3196 ),
3197 );
3198 continue;
3199 }
3200 self.state.event(
3201 "implement",
3202 format!(
3203 "{}: the reply named a command whose own CLI never confirmed the exit \
3204 status of; resuming the conversation",
3205 seat.key
3206 ),
3207 );
3208 let mut retry = job.clone();
3209 retry.seat = seat.clone();
3210 retry.prompt = prompt::resume_incomplete(
3211 "a command in your last reply had no confirmed exit status",
3212 );
3213 retry.timeout = retry_budget(job.timeout, true);
3214 retry.stem = format!("{}-confirm", job.stem);
3215 let cache = self.state.config.cache_dir();
3216 let ctx = WaveCtx {
3217 carry_seats: false,
3218 pauses: Some((&self.pause, &self.interrupt)),
3219 run: run_id,
3220 node: "implement",
3221 prompts,
3222 cache: cache.as_deref(),
3223 round: None,
3224 };
3225 let (resumed_seat, resumed) =
3226 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3227 *seat = resumed_seat;
3228 *out = resumed;
3229 }
3230 }
3231
3232 async fn continue_fix_report(
3253 &mut self,
3254 mut seat: SeatState,
3255 parse_err: String,
3256 job: &SeatJob,
3257 prompts: &Prompts,
3258 run_id: &str,
3259 round: usize,
3260 ) -> (
3261 SeatState,
3262 Option<FixReport>,
3263 Option<String>,
3264 ContinuationRecord,
3265 ) {
3266 let mut last_err = parse_err;
3267 let mut cumulative_wait_ms = 0u64;
3268 let mut attempts = 0usize;
3269 loop {
3270 if !has_context(&job.spec, &seat, job.sessions) {
3271 self.state.event(
3272 "fix",
3273 format!(
3274 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3275 session left to resume into"
3276 ),
3277 );
3278 let outcome = if attempts == 0 {
3279 ContinuationOutcome::NoSession
3280 } else {
3281 ContinuationOutcome::Exhausted
3282 };
3283 return (
3284 seat,
3285 None,
3286 Some(format!("unparsable fix report: {last_err}")),
3287 ContinuationRecord {
3288 attempts,
3289 cumulative_wait_ms,
3290 outcome,
3291 },
3292 );
3293 }
3294 if attempts >= MAX_FIX_CONTINUATIONS {
3295 self.state.event(
3296 "fix",
3297 format!(
3298 "round {round}: fixer's reply still had no adoption report after \
3299 {attempts} continuation(s) ({last_err}); giving up"
3300 ),
3301 );
3302 return (
3303 seat,
3304 None,
3305 Some(format!(
3306 "unparsable fix report after {attempts} continuation(s): {last_err}"
3307 )),
3308 ContinuationRecord {
3309 attempts,
3310 cumulative_wait_ms,
3311 outcome: ContinuationOutcome::Exhausted,
3312 },
3313 );
3314 }
3315 attempts += 1;
3316 self.state.event(
3317 "fix",
3318 format!(
3319 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3320 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3321 ),
3322 );
3323 let mut retry = job.clone();
3324 retry.seat = seat.clone();
3325 retry.prompt = prompt::resume_incomplete(&last_err);
3326 retry.timeout = retry_budget(job.timeout, true);
3327 retry.stem = format!("{}-continue{attempts}", job.stem);
3328 let cache = self.state.config.cache_dir();
3329 let ctx = WaveCtx {
3330 carry_seats: false,
3331 pauses: Some((&self.pause, &self.interrupt)),
3332 run: run_id,
3333 node: "fix",
3334 prompts,
3335 cache: cache.as_deref(),
3336 round: Some(round),
3337 };
3338 let (resumed_seat, resumed_out) = run_one(
3339 retry,
3340 Arc::clone(&self.sem),
3341 &ctx,
3342 &mut self.state,
3343 attempts,
3344 )
3345 .await;
3346 seat = resumed_seat;
3347 match resumed_out {
3348 AgentOutcome::Ok(o) => {
3349 cumulative_wait_ms += o.duration_ms;
3350 match verdict::extract_json::<FixReport>(&o.text) {
3351 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3352 self.state.event(
3353 "fix",
3354 format!(
3355 "round {round}: fixer's adoption report recovered after \
3356 {attempts} continuation(s)"
3357 ),
3358 );
3359 return (
3360 seat,
3361 Some(report),
3362 None,
3363 ContinuationRecord {
3364 attempts,
3365 cumulative_wait_ms,
3366 outcome: ContinuationOutcome::Resumed,
3367 },
3368 );
3369 }
3370 Ok(_) => {
3378 last_err = "the reply parsed, but it reported a command whose own CLI \
3379 never confirmed an exit status"
3380 .to_owned();
3381 }
3382 Err(e) => last_err = e.to_string(),
3383 }
3384 }
3385 AgentOutcome::Quota(o) => {
3386 cumulative_wait_ms += o.duration_ms;
3387 self.state.quota.push(QuotaLoss {
3388 seat: seat.key.clone(),
3389 node: "fix".to_owned(),
3390 at: Timestamp::now(),
3391 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3392 });
3393 self.state.event(
3394 "fix",
3395 format!(
3396 "round {round}: continuation rate limited (quota); not retrying now"
3397 ),
3398 );
3399 return (
3400 seat,
3401 None,
3402 Some("rate limited (quota) while recovering the fix report".to_owned()),
3403 ContinuationRecord {
3404 attempts,
3405 cumulative_wait_ms,
3406 outcome: ContinuationOutcome::QuotaLost,
3407 },
3408 );
3409 }
3410 AgentOutcome::Dropped(o) => {
3411 cumulative_wait_ms += o.duration_ms;
3412 let why = o
3413 .dropped
3414 .as_ref()
3415 .map(|d| d.why.as_str())
3416 .unwrap_or("the CLI ended the stream without delivering its answer");
3417 last_err = format!("the CLI dropped the stream ({why})");
3418 }
3419 AgentOutcome::Failed(e) => last_err = e,
3420 AgentOutcome::ResourceBlocked(e) => {
3423 return (
3424 seat,
3425 None,
3426 Some(format!("the continuation was not started: {e}")),
3427 ContinuationRecord {
3428 attempts: attempts.saturating_sub(1),
3429 cumulative_wait_ms,
3430 outcome: ContinuationOutcome::NoSession,
3431 },
3432 );
3433 }
3434 }
3435 }
3436 }
3437
3438 fn after_implement(&mut self) -> Result<()> {
3439 if !self.state.candidate_preservation.is_empty() {
3440 bail!("candidate preservation incomplete; judging is unsafe");
3441 }
3442 if self.state.leaks.is_empty() {
3444 let cfg = self.state.config.blind.clone();
3445 let mut leaks = Vec::new();
3446 for c in &self.state.candidates {
3447 let Some(patch) =
3448 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3449 else {
3450 continue;
3451 };
3452 leaks.extend(blind::scan(
3453 &format!("candidate {} patch", c.label),
3454 &patch,
3455 &cfg.vendor_tokens,
3456 ));
3457 }
3458 if !leaks.is_empty() {
3459 let summary = leaks
3460 .iter()
3461 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3462 .collect::<Vec<_>>()
3463 .join(", ");
3464 match cfg.on_leak {
3465 LeakPolicy::Fail => {
3466 self.state.status = RunStatus::Failed;
3467 self.state
3468 .event("blind", format!("vendor text in a patch: {summary}"));
3469 self.state.leaks = leaks;
3470 self.state.save()?;
3471 self.settle_questions();
3472 bail!(
3473 "blind.on_leak = \"fail\" and vendor text reached a \
3474 judged patch: {summary}"
3475 );
3476 }
3477 LeakPolicy::Redact => self.state.event(
3478 "blind",
3479 format!("redacting vendor text for judging: {summary}"),
3480 ),
3481 LeakPolicy::Warn => self.state.event(
3482 "blind",
3483 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3484 ),
3485 }
3486 self.state.leaks = leaks;
3487 }
3488 }
3489
3490 if self.state.viable().is_empty() {
3491 if self.state.all_candidates_verified_noop() {
3492 self.state.status = RunStatus::VerifiedNoop;
3503 self.state.save()?;
3504 self.settle_questions();
3505 return Ok(());
3506 }
3507 self.state.status = RunStatus::Failed;
3508 self.state.save()?;
3509 self.settle_questions();
3510 bail!("no candidate produced a change; nothing to judge");
3511 }
3512 self.state.status = RunStatus::Judging;
3513 self.state.save()?;
3514 Ok(())
3515 }
3516
3517 async fn judge(&mut self) -> Result<()> {
3520 let run_id = self.state.id.clone();
3525 let prompts = self.state.config.prompts.clone();
3526 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3527 return Ok(());
3528 }
3529 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3530 if viable.len() == 1 {
3531 self.state.judge_skipped = true;
3538 self.state.event(
3539 "judge",
3540 format!(
3541 "only candidate {} produced a change; judging skipped",
3542 viable[0].label
3543 ),
3544 );
3545 self.state.save()?;
3546 return Ok(());
3547 }
3548 self.state.status = RunStatus::Judging;
3549
3550 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3551 let language = self.state.config.graph.language.clone();
3552 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3553 let sessions = self.state.config.graph.sessions;
3554 let artifacts = agent::artifacts_dir(&self.state.dir());
3555 let root = self.state.worktree_root();
3556 let base_short = short(&self.state.base_commit);
3557
3558 let mut jobs = Vec::new();
3559 let mut orders = Vec::new();
3560 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3561 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3562 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3563 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3564 let seat_key = format!("judge-{}", j + 1);
3565 let seat = self.seat(&seat_key, &spec.id);
3566 jobs.push(SeatJob {
3567 prompt: prompt::judge(
3568 &self.state.instruction,
3569 &views,
3570 self.roles.judges.len(),
3571 &base_short,
3572 &language,
3573 ),
3574 spec,
3575 seat,
3576 cwd: root.join(format!("judge-{}", j + 1)),
3577 timeout,
3578 allow_write: false,
3579 sessions,
3580 artifacts: artifacts.clone(),
3581 stem: format!("judge-{}", j + 1),
3582 handover: None,
3583 });
3584 }
3585
3586 self.state.event(
3587 "judge",
3588 format!(
3589 "{} judges ranking {} candidates blind",
3590 jobs.len(),
3591 viable.len()
3592 ),
3593 );
3594 let labels_for_check = labels.clone();
3595 let mut quota_losses = Vec::new();
3596 let cache = self.state.config.cache_dir();
3597 let ctx = WaveCtx {
3598 carry_seats: false,
3599 pauses: Some((&self.pause, &self.interrupt)),
3600 run: &run_id,
3601 node: "judge",
3602 prompts: &prompts,
3603 cache: cache.as_deref(),
3604 round: None,
3605 };
3606 let results = ask_json_wave::<Ranking>(
3607 jobs,
3608 Arc::clone(&self.sem),
3609 self.state.config.graph.retries,
3610 &self.roles.judge_roster,
3611 &ctx,
3612 &mut quota_losses,
3613 &mut self.state,
3614 &move |r: &Ranking| r.validate(&labels_for_check),
3615 )
3616 .await;
3617 if self.state.resource_block.is_some() {
3618 return self.stop_for_resource();
3619 }
3620 self.state.quota.extend(quota_losses);
3621
3622 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3623 let agent_id = seat.agent.clone();
3624 self.state.seats.insert(seat.key.clone(), seat);
3625 let mut record = Judgement {
3626 judge: j + 1,
3627 seat: format!("judge-{}", j + 1),
3628 agent: agent_id,
3629 ranking: Vec::new(),
3630 reasons: BTreeMap::new(),
3631 confidence: None,
3632 strengths: BTreeMap::new(),
3633 order: orders[j].clone(),
3634 failed: None,
3635 duration_ms: 0,
3636 };
3637 match res {
3638 Ok((ranking, out)) => {
3639 record.ranking = ranking.normalized();
3640 record.reasons = ranking.reasons;
3641 record.strengths = ranking.strengths;
3642 record.confidence = ranking.confidence;
3643 record.duration_ms = out.duration_ms;
3644 self.state.event(
3645 "judge",
3646 format!(
3647 "judge {} ranked {}",
3648 j + 1,
3649 record.ranking.iter().collect::<String>()
3650 ),
3651 );
3652 }
3653 Err(e) => {
3654 record.failed = Some(e.to_string());
3655 self.state
3656 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3657 }
3658 }
3659 self.state.judgements.push(record);
3660 self.state.save()?;
3661 }
3662 Ok(())
3663 }
3664
3665 async fn deliberate(&mut self) -> Result<()> {
3668 if self.hold_base_sync().await? {
3669 return Ok(());
3670 }
3671 let run_id = self.state.id.clone();
3676 let prompts = self.state.config.prompts.clone();
3677 if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3678 return Ok(());
3679 }
3680 let tops: Vec<char> = self
3681 .state
3682 .judgements
3683 .iter()
3684 .filter_map(|j| j.ranking.first().copied())
3685 .collect();
3686 let rounds = self.state.config.graph.deliberate_rounds;
3687 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3688 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3689 self.state.event(
3690 "deliberate",
3691 format!("judges agreed on {} outright; no deliberation", tops[0]),
3692 );
3693 }
3694 self.state.status = RunStatus::Voting;
3695 self.state.save()?;
3696 return Ok(());
3697 }
3698
3699 self.state.status = RunStatus::Deliberating;
3700 self.state.event(
3701 "deliberate",
3702 format!(
3703 "split: first choices were {} — opening {rounds} round(s)",
3704 tops.iter().collect::<String>()
3705 ),
3706 );
3707
3708 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3709 let language = self.state.config.graph.language.clone();
3710 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3711 let sessions = self.state.config.graph.sessions;
3712 let artifacts = agent::artifacts_dir(&self.state.dir());
3713 let root = self.state.worktree_root();
3714 let base_short = short(&self.state.base_commit);
3715
3716 for round in 1..=rounds {
3720 let mut turns: Vec<DeliberationTurn> = Vec::new();
3721 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3722 if self.state.judgements[j].failed.is_some() {
3723 continue;
3724 }
3725 let seat_key = format!("judge-{}", j + 1);
3726 let spec = self.occupant(&seat_key, spec);
3727 let mut seat = self.seat(&seat_key, &spec.id);
3728 let transcript = self.transcript(&turns, j);
3729 let build = |context: Option<&str>| {
3730 prompt::deliberate(
3731 &self.state.instruction,
3732 context,
3733 &transcript,
3734 round,
3735 rounds,
3736 &language,
3737 )
3738 };
3739 let block = self.candidate_block(&viable, &base_short);
3740 let full = build(Some(&block));
3741 let text = if has_context(&spec, &seat, sessions) {
3742 build(None)
3743 } else {
3744 full.clone()
3745 };
3746 let job = SeatJob {
3747 spec,
3748 seat: seat.clone(),
3749 prompt: text,
3750 cwd: root.join(format!("judge-{}", j + 1)),
3751 timeout,
3752 allow_write: false,
3753 sessions,
3754 artifacts: artifacts.clone(),
3755 stem: format!("delib-{round}-judge-{}", j + 1),
3756 handover: Some(full),
3757 };
3758 let cache = self.state.config.cache_dir();
3759 let ctx = WaveCtx {
3760 carry_seats: false,
3761 pauses: Some((&self.pause, &self.interrupt)),
3762 run: &run_id,
3763 node: "deliberate",
3764 prompts: &prompts,
3765 cache: cache.as_deref(),
3766 round: None,
3767 };
3768 let mut losses = Vec::new();
3772 let mut results = ask_wave_with::<String>(
3773 vec![job],
3774 Arc::clone(&self.sem),
3775 0,
3776 &self.roles.judge_roster,
3777 &ctx,
3778 &mut losses,
3779 &mut self.state,
3780 &|text: &str| {
3781 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3782 },
3783 )
3784 .await;
3785 self.state.quota.extend(losses);
3786 let (updated, res, _) = results.pop().expect("one job in, one result out");
3787 seat = updated;
3788 let agent_id = seat.agent.clone();
3789 self.state.seats.insert(seat.key.clone(), seat);
3790 let body = match res {
3791 Ok((body, _)) => body,
3792 Err(e) => {
3795 self.state
3796 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3797 continue;
3798 }
3799 };
3800 let tentative = verdict::extract_json::<Position>(&body)
3801 .ok()
3802 .and_then(|p| p.tentative)
3803 .and_then(|s| s.trim().chars().next())
3804 .map(|c| c.to_ascii_uppercase());
3805 self.state.event(
3806 "deliberate",
3807 format!(
3808 "round {round}: judge {} now favours {}",
3809 j + 1,
3810 tentative.map_or("—".to_owned(), |c| c.to_string())
3811 ),
3812 );
3813 turns.push(DeliberationTurn {
3814 judge: j + 1,
3815 agent: agent_id,
3816 body: blind::sanitize_prose(&body, &self.state.config.blind),
3817 tentative,
3818 });
3819 }
3820 self.state
3821 .deliberation
3822 .push(DeliberationRound { round, turns });
3823 self.state.save()?;
3824 }
3825
3826 self.state.status = RunStatus::Voting;
3827 self.state.save()?;
3828 Ok(())
3829 }
3830
3831 async fn vote(&mut self) -> Result<()> {
3834 if self.hold_base_sync().await? {
3835 return Ok(());
3836 }
3837 let run_id = self.state.id.clone();
3842 let prompts = self.state.config.prompts.clone();
3843 if !self.state.votes.is_empty() || self.state.tally.is_some() {
3844 return Ok(());
3845 }
3846 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3847 if viable.len() == 1 {
3848 return Ok(());
3849 }
3850 self.state.status = RunStatus::Voting;
3851
3852 let language = self.state.config.graph.language.clone();
3853 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3854 let sessions = self.state.config.graph.sessions;
3855 let artifacts = agent::artifacts_dir(&self.state.dir());
3856 let root = self.state.worktree_root();
3857 let base_short = short(&self.state.base_commit);
3858 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3859
3860 let mut jobs = Vec::new();
3861 let mut seats_at = Vec::new();
3862 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3863 if self
3864 .state
3865 .judgements
3866 .get(j)
3867 .is_some_and(|r| r.failed.is_some())
3868 {
3869 continue;
3870 }
3871 let seat_key = format!("judge-{}", j + 1);
3872 let spec = self.occupant(&seat_key, spec);
3873 let seat = self.seat(&seat_key, &spec.id);
3874 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3875 let text = if has_context(&spec, &seat, sessions) {
3876 prompt::final_vote(&viable, &language)
3877 } else {
3878 full.clone()
3879 };
3880 jobs.push(SeatJob {
3881 spec,
3882 seat,
3883 prompt: text,
3884 cwd: root.join(format!("judge-{}", j + 1)),
3885 timeout,
3886 allow_write: false,
3887 sessions,
3888 artifacts: artifacts.clone(),
3889 stem: format!("vote-judge-{}", j + 1),
3890 handover: Some(full),
3891 });
3892 seats_at.push(j);
3893 }
3894
3895 self.state.event(
3896 "vote",
3897 format!(
3898 "collecting {} final votes one by one, privately",
3899 jobs.len()
3900 ),
3901 );
3902 let allowed = viable.clone();
3903 let mut quota_losses = Vec::new();
3904 let cache = self.state.config.cache_dir();
3905 let ctx = WaveCtx {
3906 carry_seats: false,
3907 pauses: Some((&self.pause, &self.interrupt)),
3908 run: &run_id,
3909 node: "vote",
3910 prompts: &prompts,
3911 cache: cache.as_deref(),
3912 round: None,
3913 };
3914 let results = ask_json_wave::<FinalVote>(
3915 jobs,
3916 Arc::clone(&self.sem),
3917 self.state.config.graph.retries,
3918 &self.roles.judge_roster,
3919 &ctx,
3920 &mut quota_losses,
3921 &mut self.state,
3922 &move |v: &FinalVote| match v.label() {
3923 Some(c) if allowed.contains(&c) => Ok(()),
3924 other => bail!("vote {other:?} is not one of {allowed:?}"),
3925 },
3926 )
3927 .await;
3928 if self.state.resource_block.is_some() {
3929 return self.stop_for_resource();
3930 }
3931 self.state.quota.extend(quota_losses);
3932
3933 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3934 let agent_id = seat.agent.clone();
3935 self.state.seats.insert(seat.key.clone(), seat);
3936 let initial = self
3937 .state
3938 .judgements
3939 .get(j)
3940 .and_then(|r| r.ranking.first().copied());
3941 let mut record = VoteRecord {
3942 judge: j + 1,
3943 agent: agent_id,
3944 vote: None,
3945 reason: String::new(),
3946 changed: false,
3947 };
3948 match res {
3949 Ok((v, _)) => {
3950 record.vote = v.label();
3951 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3952 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3953 self.state.event(
3954 "vote",
3955 format!(
3956 "judge {} voted {}{}",
3957 j + 1,
3958 record.vote.unwrap_or('?'),
3959 if record.changed { " (changed)" } else { "" }
3960 ),
3961 );
3962 }
3963 Err(e) => {
3964 self.state
3965 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3966 }
3967 }
3968 self.state.votes.push(record);
3969 self.state.save()?;
3970 }
3971 Ok(())
3972 }
3973
3974 fn tally(&mut self) -> Result<()> {
3977 if self.state.tally.is_some() {
3978 return Ok(());
3979 }
3980 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3981 let tops: Vec<char> = self
3982 .state
3983 .judgements
3984 .iter()
3985 .filter_map(|j| j.ranking.first().copied())
3986 .collect();
3987 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3988
3989 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3992 let mut cast: Vec<char> = Vec::new();
3993 for (i, j) in self.state.judgements.iter().enumerate() {
3994 let vote = self
3995 .state
3996 .votes
3997 .iter()
3998 .find(|v| v.judge == i + 1)
3999 .and_then(|v| v.vote)
4000 .or_else(|| j.ranking.first().copied());
4001 if let Some(v) = vote {
4002 *first_choice.entry(v).or_insert(0) += 1;
4003 cast.push(v);
4004 }
4005 }
4006
4007 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4008 for j in &self.state.judgements {
4009 let n = j.ranking.len();
4010 for (pos, label) in j.ranking.iter().enumerate() {
4011 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4012 }
4013 }
4014
4015 let best = first_choice.values().copied().max().unwrap_or(0);
4016 let mut leaders: Vec<char> = first_choice
4017 .iter()
4018 .filter(|(_, v)| **v == best)
4019 .map(|(k, _)| *k)
4020 .collect();
4021 let mut tie_break = None;
4022 if leaders.len() > 1 {
4023 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4024 let borda_leaders: Vec<char> = leaders
4025 .iter()
4026 .copied()
4027 .filter(|l| borda[l] == top_borda)
4028 .collect();
4029 tie_break = Some(if borda_leaders.len() == 1 {
4030 format!(
4031 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4032 leaders.len()
4033 )
4034 } else {
4035 format!(
4036 "{} way tie on both first-choice votes and Borda points, broken by label order",
4037 leaders.len()
4038 )
4039 });
4040 leaders = borda_leaders;
4041 leaders.sort_unstable();
4042 }
4043 let winner = *leaders
4044 .first()
4045 .or(viable.first())
4046 .context("no candidate to declare a winner from")?;
4047
4048 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4049 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4050 let deliberated = !self.state.deliberation.is_empty();
4051
4052 let quota_seats: std::collections::BTreeSet<&str> =
4056 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4057 let mut present = 0usize;
4058 for (i, j) in self.state.judgements.iter().enumerate() {
4059 if quota_seats.contains(j.seat.as_str()) {
4060 continue;
4061 }
4062 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4063 let voted = self
4064 .state
4065 .votes
4066 .iter()
4067 .any(|v| v.judge == i + 1 && v.vote.is_some());
4068 if ranked || voted {
4069 present += 1;
4070 }
4071 }
4072 let needs_quorum = viable.len() > 1;
4078 let judges_total = if needs_quorum {
4079 self.roles.judges.len()
4080 } else {
4081 0
4082 };
4083 let quorum = if needs_quorum {
4084 judges_total / 2 + 1
4085 } else {
4086 0
4087 };
4088 let met_quorum = !needs_quorum || present >= quorum;
4089 let uncontested = (!needs_quorum).then(|| {
4090 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4091 });
4092
4093 self.state.event(
4094 "tally",
4095 match &uncontested {
4096 Some(reason) => format!("winner {winner} — {reason}"),
4097 None => format!(
4098 "winner {winner} — votes {} | initial {} | {} changed | \
4099 {present}/{judges_total} judges{}",
4100 first_choice
4101 .iter()
4102 .map(|(k, v)| format!("{k}:{v}"))
4103 .collect::<Vec<_>>()
4104 .join(" "),
4105 if unanimous_initial {
4106 "unanimous"
4107 } else {
4108 "split"
4109 },
4110 changed_votes,
4111 if met_quorum {
4112 String::new()
4113 } else {
4114 format!(" — below quorum ({quorum} required)")
4115 },
4116 ),
4117 },
4118 );
4119 if !met_quorum {
4120 self.state.event(
4121 "stall",
4122 format!(
4123 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4124 the run stops here, resumable"
4125 ),
4126 );
4127 }
4128 self.state.tally = Some(Tally {
4129 first_choice,
4130 borda,
4131 winner,
4132 rankings: tops.len(),
4133 unanimous_initial,
4134 deliberated,
4135 changed_votes,
4136 unanimous_final,
4137 tie_break,
4138 judges: judges_total,
4139 present,
4140 quorum,
4141 met_quorum,
4142 uncontested,
4143 });
4144 self.state.status = if met_quorum {
4145 RunStatus::Reviewing
4146 } else {
4147 RunStatus::Stalled
4148 };
4149 self.state.save()?;
4150 Ok(())
4151 }
4152
4153 #[allow(clippy::too_many_lines)]
4174 async fn recover_stall(&mut self) -> Result<bool> {
4175 let run_id = self.state.id.clone();
4180 let prompts = self.state.config.prompts.clone();
4181 let quota_seats: BTreeSet<&str> =
4186 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4187 let absent: Vec<String> = self
4188 .state
4189 .judgements
4190 .iter()
4191 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4192 .map(|j| j.seat.clone())
4193 .collect();
4194 if absent.is_empty() {
4195 return Ok(false);
4196 }
4197 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4198 if viable.len() <= 1 {
4199 return Ok(false);
4200 }
4201 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4202 let language = self.state.config.graph.language.clone();
4203 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4204 let sessions = self.state.config.graph.sessions;
4205 let artifacts = agent::artifacts_dir(&self.state.dir());
4206 let root = self.state.worktree_root();
4207 let base_short = short(&self.state.base_commit);
4208 let candidates: Vec<Candidate> = viable.clone();
4209
4210 let mut positions: Vec<usize> = absent
4212 .iter()
4213 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4214 .collect();
4215 if positions.is_empty() {
4216 return Ok(false);
4217 }
4218 positions.sort_unstable();
4219 positions.dedup();
4220
4221 let mut judge_jobs = Vec::new();
4223 for &j in &positions {
4224 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4225 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4226 let seat_key = format!("judge-{}", j + 1);
4227 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4228 let seat = self.seat(&seat_key, &spec.id);
4229 judge_jobs.push(SeatJob {
4230 spec,
4231 seat,
4232 prompt: prompt::judge(
4233 &self.state.instruction,
4234 &views,
4235 self.roles.judges.len(),
4236 &base_short,
4237 &language,
4238 ),
4239 cwd: root.join(seat_key),
4240 timeout,
4241 allow_write: false,
4242 sessions,
4243 artifacts: artifacts.clone(),
4244 stem: format!("judge-{}-recover", j + 1),
4245 handover: None,
4246 });
4247 }
4248
4249 let labels_for_check = labels.clone();
4250 let mut judge_losses = Vec::new();
4251 let retries = self.state.config.graph.retries;
4252 let cache = self.state.config.cache_dir();
4253 let ctx = WaveCtx {
4254 carry_seats: false,
4255 pauses: Some((&self.pause, &self.interrupt)),
4256 run: &run_id,
4257 node: "judge",
4258 prompts: &prompts,
4259 cache: cache.as_deref(),
4260 round: None,
4261 };
4262 let results = ask_json_wave::<Ranking>(
4263 judge_jobs,
4264 Arc::clone(&self.sem),
4265 retries,
4266 &self.roles.judge_roster,
4267 &ctx,
4268 &mut judge_losses,
4269 &mut self.state,
4270 &move |r: &Ranking| r.validate(&labels_for_check),
4271 )
4272 .await;
4273 if self.state.resource_block.is_some() {
4274 self.stop_for_resource()?;
4275 return Ok(false);
4276 }
4277
4278 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4280 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4281 let agent_id = seat.agent.clone();
4282 self.state.seats.insert(seat.key.clone(), seat);
4283 let record = &mut self.state.judgements[j];
4284 match res {
4285 Ok((ranking, out)) => {
4286 record.agent = agent_id;
4287 record.ranking = ranking.normalized();
4288 record.reasons = ranking.reasons;
4289 record.strengths = ranking.strengths;
4290 record.confidence = ranking.confidence;
4291 record.failed = None;
4292 record.duration_ms = out.duration_ms;
4293 recovered.insert(j);
4294 self.state.event(
4295 "recover",
4296 format!("judge {} ranked again after the limit", j + 1),
4297 );
4298 }
4299 Err(e) => {
4300 self.state
4301 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4302 }
4303 }
4304 }
4305
4306 let mut vote_jobs = Vec::new();
4308 let mut vote_pos: Vec<usize> = Vec::new();
4309 for &j in &recovered {
4310 let seat_key = format!("judge-{}", j + 1);
4311 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4312 let seat = self.seat(&seat_key, &spec.id);
4313 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4314 let text = if has_context(&spec, &seat, sessions) {
4315 prompt::final_vote(&labels, &language)
4316 } else {
4317 full.clone()
4318 };
4319 vote_jobs.push(SeatJob {
4320 spec,
4321 seat,
4322 prompt: text,
4323 cwd: root.join(seat_key),
4324 timeout,
4325 allow_write: false,
4326 sessions,
4327 artifacts: artifacts.clone(),
4328 stem: format!("vote-judge-{}-recover", j + 1),
4329 handover: Some(full),
4330 });
4331 vote_pos.push(j);
4332 }
4333 let allowed = labels.clone();
4334 let mut vote_losses = Vec::new();
4335 let vote_retries = self.state.config.graph.retries;
4336 let vote_cache = self.state.config.cache_dir();
4337 let ctx = WaveCtx {
4338 carry_seats: false,
4339 pauses: Some((&self.pause, &self.interrupt)),
4340 run: &run_id,
4341 node: "vote",
4342 prompts: &prompts,
4343 cache: vote_cache.as_deref(),
4344 round: None,
4345 };
4346 let votes = ask_json_wave::<FinalVote>(
4347 vote_jobs,
4348 Arc::clone(&self.sem),
4349 vote_retries,
4350 &self.roles.judge_roster,
4351 &ctx,
4352 &mut vote_losses,
4353 &mut self.state,
4354 &move |v: &FinalVote| match v.label() {
4355 Some(c) if allowed.contains(&c) => Ok(()),
4356 other => bail!("vote {other:?} is not one of {allowed:?}"),
4357 },
4358 )
4359 .await;
4360 if self.state.resource_block.is_some() {
4361 self.stop_for_resource()?;
4362 return Ok(false);
4363 }
4364 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4365 let agent_id = seat.agent.clone();
4366 self.state.seats.insert(seat.key.clone(), seat);
4367 match res {
4368 Ok((v, _)) => {
4369 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4370 rec.vote = v.label();
4371 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4372 } else {
4373 self.state.votes.push(VoteRecord {
4374 judge: j + 1,
4375 agent: agent_id,
4376 vote: v.label(),
4377 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4378 changed: false,
4379 });
4380 }
4381 self.state.event(
4382 "recover",
4383 format!("judge {} voted again after the limit", j + 1),
4384 );
4385 }
4386 Err(e) => {
4387 self.state
4388 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4389 }
4390 }
4391 }
4392
4393 let recovered_keys: BTreeSet<String> = recovered
4397 .iter()
4398 .map(|&j| format!("judge-{}", j + 1))
4399 .collect();
4400 self.state
4401 .quota
4402 .retain(|q| !recovered_keys.contains(&q.seat));
4403 for loss in judge_losses.into_iter().chain(vote_losses) {
4407 if recovered_keys.contains(&loss.seat) {
4408 continue;
4409 }
4410 self.state.quota.retain(|q| q.seat != loss.seat);
4411 self.state.quota.push(loss);
4412 }
4413
4414 self.state.tally = None;
4416 self.tally()?;
4417 Ok(self
4418 .state
4419 .tally
4420 .as_ref()
4421 .map(|t| t.met_quorum)
4422 .unwrap_or(false))
4423 }
4424
4425 async fn graft(&mut self) -> Result<()> {
4441 let Err(e) = self.graft_inner().await else {
4442 return Ok(());
4443 };
4444 if e.is::<crate::repo_lock::ResourceBlocked>() {
4445 return Err(e);
4446 }
4447 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4448 self.state.status = RunStatus::Blocked;
4449 self.state.event("graft", format!("blocked: {e:#}"));
4450 self.state.save()?;
4451 return Err(e);
4452 }
4453 if let Some(rec) = self.state.graft.clone()
4458 && !rec.outcome.settled()
4459 {
4460 if let Some(winner) = self.state.winner().cloned() {
4461 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4462 if re.is::<crate::repo_lock::ResourceBlocked>() {
4463 return Err(re);
4464 }
4465 self.state.status = RunStatus::Blocked;
4466 self.state.event("graft", format!("blocked: {re:#}"));
4467 self.state.save()?;
4468 return Err(re);
4469 }
4470 }
4471 if let Some(g) = self.state.graft.as_mut() {
4472 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4473 }
4474 }
4475 self.state.event("graft", format!("skipped: {e:#}"));
4476 if let Err(e) = self.state.save() {
4477 tracing::warn!("save after graft: {e:#}");
4478 }
4479 Ok(())
4480 }
4481
4482 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4485 let _coordination = self.coordinate(wt).await?;
4486 crate::recovery::restore_repair(wt, pre)
4487 .await
4488 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4489 }
4490
4491 fn graft_skipped(&mut self, why: &str) {
4493 let message = format!("skipped: {why}");
4494 if !self
4495 .state
4496 .events
4497 .iter()
4498 .any(|e| e.node == "graft" && e.message == message)
4499 {
4500 self.state.event("graft", message);
4501 }
4502 }
4503
4504 async fn graft_inner(&mut self) -> Result<()> {
4505 if let Some(rec) = self.state.graft.clone()
4508 && !rec.outcome.settled()
4509 {
4510 if self.state.reviews.is_empty()
4514 && let Some(winner) = self.state.winner().cloned()
4515 {
4516 self.graft_restore(&winner.worktree, &rec.pre).await?;
4517 }
4518 if rec.outcome == GraftOutcome::WaitingForRestore {
4519 self.state.graft = None;
4520 self.state.save()?;
4521 } else {
4522 if let Some(g) = self.state.graft.as_mut() {
4523 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4524 }
4525 self.state.event(
4526 "graft",
4527 "rolled back: the previous round was interrupted; the winner is untouched",
4528 );
4529 self.state.save()?;
4530 return Ok(());
4531 }
4532 }
4533 let plan = match graft::plan(&self.state) {
4534 Ok(plan) => plan,
4535 Err(skip) => {
4536 if skip != graft::Skip::AlreadyDone {
4537 self.graft_skipped(skip.reason());
4538 self.state.save()?;
4539 }
4540 return Ok(());
4541 }
4542 };
4543 let Some(winner) = self.state.winner().cloned() else {
4544 return Ok(());
4545 };
4546 let wt = winner.worktree.clone();
4547 if !git::is_clean(&wt).await? {
4548 self.graft_skipped("the winner's worktree has uncommitted changes");
4549 self.state.save()?;
4550 return Ok(());
4551 }
4552 let pre = git::rev_parse(&wt, "HEAD").await?;
4553 self.state.graft = Some(GraftRecord {
4556 pre: pre.clone(),
4557 post: None,
4558 lines: None,
4559 limit: None,
4560 agent: None,
4561 outcome: GraftOutcome::Running,
4562 requested: graft::requested(&plan),
4563 adoption: None,
4564 adoption_error: None,
4565 });
4566 self.state.event(
4567 "graft",
4568 format!(
4569 "porting strengths of candidate(s) {} onto {}",
4570 plan.strengths
4571 .iter()
4572 .map(|(l, _, _)| l.to_string())
4573 .collect::<Vec<_>>()
4574 .join(", "),
4575 plan.winner
4576 ),
4577 );
4578 self.state.save()?;
4579
4580 let fix_prompt = prompt::graft(
4581 &self.state.instruction,
4582 plan.winner,
4583 &plan.strengths,
4584 &self.state.config.graph.language,
4585 );
4586 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4587 let sessions = self.state.config.graph.sessions;
4588 let artifacts = agent::artifacts_dir(&self.state.dir());
4589 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4590 let (_, seat, out) = self
4591 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4592 SeatJob {
4593 prompt: fix_prompt.clone(),
4594 spec,
4595 seat,
4596 cwd: wt.clone(),
4597 timeout,
4598 allow_write: true,
4599 sessions,
4600 artifacts: artifacts.clone(),
4601 stem: "graft".to_owned(),
4602 handover: None,
4603 }
4604 })
4605 .await;
4606 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4607 self.state.graft.as_mut().expect("graft record").outcome =
4610 GraftOutcome::WaitingForRestore;
4611 return self.stop_for_resource();
4612 }
4613 let mut why: Option<String> = None;
4614 match &out {
4615 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4616 why = Some("the fixer produced nothing usable".to_owned());
4617 }
4618 AgentOutcome::Ok(_) => {}
4619 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4620 AgentOutcome::Quota(o) => {
4621 self.state.quota.push(QuotaLoss {
4622 seat: seat.key.clone(),
4623 node: "graft".to_owned(),
4624 at: Timestamp::now(),
4625 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4626 });
4627 why = Some("rate limited (quota)".to_owned());
4628 }
4629 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4630 why = Some(format!(
4631 "the fixer failed: {}",
4632 e.lines()
4633 .next()
4634 .unwrap_or("")
4635 .chars()
4636 .take(160)
4637 .collect::<String>()
4638 ));
4639 }
4640 }
4641 let agent_id = seat.agent.clone();
4642 self.state.seats.insert(seat.key.clone(), seat);
4643
4644 let mut post = None;
4645 let mut lines = None;
4646 let _coordination = self.coordinate(&wt).await?;
4647 if why.is_none() {
4648 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4649 if let Ok(r) = &rescued {
4650 self.state.note_withheld("graft", &r.withheld);
4651 }
4652 let head = git::rev_parse(&wt, "HEAD").await?;
4653 let dirty = match &rescued {
4657 Ok(r) => git::status_porcelain(&wt)
4658 .await
4659 .map(|st| {
4660 st.lines().any(|l| {
4661 !r.withheld
4662 .iter()
4663 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4664 })
4665 })
4666 .unwrap_or(true),
4667 Err(_) => true,
4668 };
4669 if dirty {
4670 why = Some("the fixer left changes that could not be committed".to_owned());
4671 } else if head == pre {
4672 why = Some("the fixer changed nothing".to_owned());
4673 } else {
4674 lines = git::changed_lines(&wt, &pre, &head).await;
4675 post = Some(head);
4676 }
4677 }
4678 let (adoption, adoption_error) = match &out {
4679 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4680 _ => (None, None),
4681 };
4682 let requested = graft::requested(&plan);
4683 let outcome = match why {
4684 None => {
4685 self.state.event(
4686 "graft",
4687 format!(
4688 "applied: {}",
4689 lines.map_or_else(
4690 || "changed lines unavailable".to_owned(),
4691 |n| format!("{n} changed lines")
4692 )
4693 ),
4694 );
4695 GraftOutcome::Applied
4696 }
4697 Some(why) => {
4698 crate::recovery::restore_repair(&wt, &pre)
4701 .await
4702 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4703 self.state.event(
4704 "graft",
4705 format!("rolled back: {why}; the winner is untouched"),
4706 );
4707 GraftOutcome::RolledBack(why)
4708 }
4709 };
4710 self.state.graft = Some(GraftRecord {
4711 pre,
4712 post,
4713 lines,
4714 limit: None,
4715 agent: Some(agent_id),
4716 outcome,
4717 requested,
4718 adoption,
4719 adoption_error,
4720 });
4721 self.state.save()?;
4722 Ok(())
4723 }
4724
4725 async fn fold_losers(&mut self) -> Result<()> {
4728 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4729 return Ok(());
4730 };
4731 if !self.state.candidate_preservation.is_empty() {
4732 bail!("candidate preservation incomplete; refusing cleanup");
4733 }
4734 let repo = self.state.repo.clone();
4735 let _coordination = self.coordinate(&repo).await?;
4736 let mut folded = Vec::new();
4737 for i in 0..self.state.candidates.len() {
4738 let c = &self.state.candidates[i];
4739 if c.label == winner || c.folded {
4740 continue;
4741 }
4742 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4743 remove_preserved_worktree(&repo, &wt).await?;
4744 if git::branch_exists(&repo, &branch).await?
4745 && !git::branch_delete(&repo, &branch).await?
4746 {
4747 bail!("candidate branch removal refused");
4748 }
4749 self.state.candidates[i].folded = true;
4750 self.state.save()?;
4751 folded.push(label.to_string());
4752 }
4753 let root = self.state.worktree_root();
4755 for j in 1..=self.roles.judges.len() {
4756 let wt = root.join(format!("judge-{j}"));
4757 if wt.exists() {
4758 remove_preserved_worktree(&repo, &wt).await?;
4759 }
4760 }
4761 if self.state.config.graph.advise {
4764 for k in 1..=self.state.config.graph.advisors {
4765 let wt = root.join(format!("advisor-{k}"));
4766 if wt.exists() {
4767 remove_preserved_worktree(&repo, &wt).await?;
4768 }
4769 }
4770 }
4771 if !folded.is_empty() {
4772 self.state
4773 .event("fold", format!("folded candidates {}", folded.join(", ")));
4774 self.state.save()?;
4775 }
4776 Ok(())
4777 }
4778
4779 async fn hold_base_sync(&mut self) -> Result<bool> {
4782 self.hold_base_sync_as(None).await
4783 }
4784
4785 async fn hold_base_sync_kind(
4786 &mut self,
4787 kind: crate::base_sync_recovery::StopKind,
4788 ) -> Result<()> {
4789 self.hold_base_sync_as(Some(kind)).await?;
4790 Ok(())
4791 }
4792
4793 async fn hold_base_sync_as(
4794 &mut self,
4795 kind: Option<crate::base_sync_recovery::StopKind>,
4796 ) -> Result<bool> {
4797 if !self.state.base_sync_blocked() {
4798 self.state.base_sync_recovery.stop = None;
4799 return Ok(false);
4800 }
4801 let head = match self.state.winner() {
4802 Some(w) => git::rev_parse(&w.worktree, "HEAD")
4803 .await
4804 .unwrap_or_default(),
4805 None => String::new(),
4806 };
4807 self.state.record_base_sync_stop(&head);
4808 if let Some(kind) = kind {
4809 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4810 }
4811 self.state.save()?;
4812 Ok(true)
4813 }
4814
4815 async fn sync_to_base(&mut self) -> Result<()> {
4857 if self.state.status == RunStatus::AlreadyInBase {
4858 return Ok(());
4859 }
4860 self.hold_base_sync().await?;
4861 let conflicted = self
4862 .state
4863 .base_sync
4864 .as_ref()
4865 .is_some_and(|s| s.conflict.is_some());
4866 let Some(winner) = self.state.winner().cloned() else {
4867 return Ok(());
4868 };
4869
4870 let repo = self.state.repo.clone();
4871 let remote = self.state.config.merge.remote.clone();
4872 let base_branch = self.state.base_branch.clone();
4873 let tracking = format!("{remote}/{base_branch}");
4874
4875 let fetched_base =
4876 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4877 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4882 return Ok(());
4883 };
4884
4885 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4886 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4887 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4888
4889 if (behind > 0 || conflicted || head == tip)
4898 && self
4899 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4900 .await?
4901 {
4902 return Ok(());
4903 }
4904 if conflicted && self.state.base_recovery_pending() {
4905 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4906 let owner =
4907 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4908 if !fetched_base
4909 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4910 || !git::is_clean(&winner.worktree).await?
4911 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4912 {
4913 return Ok(());
4914 }
4915 }
4916 let recovery = conflicted && self.state.consume_base_recovery(&head);
4917 if conflicted {
4918 self.state.save()?;
4920 if !recovery {
4921 return Ok(());
4922 }
4923 }
4924
4925 if behind == 0 {
4926 if let Some(from) = self
4935 .state
4936 .rebase_fixes
4937 .iter()
4938 .rev()
4939 .find_map(|r| r.from.clone())
4940 && from != head
4941 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4942 .await
4943 .is_ok_and(|o| o.ok())
4944 {
4945 git::sync_to_head(&winner.worktree).await?;
4946 }
4947 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4953 if let Some(why) = &conflict {
4954 self.state.status = RunStatus::Blocked;
4955 self.state.event("land", why.clone());
4956 }
4957 self.state.base_sync = Some(BaseSync {
4958 tip,
4959 behind: 0,
4960 attempts: attempts + usize::from(recovery),
4961 conflict,
4962 already_in: None,
4963 });
4964 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4965 .await?;
4966 self.state.save()?;
4967 return Ok(());
4968 }
4969
4970 if attempts >= BASE_SYNC_ROUNDS && !recovery {
4971 let why = format!(
4972 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4973 synchronization(s); synchronizing again would only race it",
4974 winner.branch
4975 );
4976 self.state.status = RunStatus::Blocked;
4977 self.state.base_sync = Some(BaseSync {
4978 tip,
4979 behind,
4980 attempts,
4981 conflict: Some(why.clone()),
4982 already_in: None,
4983 });
4984 self.state.event("land", why);
4985 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4986 .await?;
4987 self.state.save()?;
4988 return Ok(());
4989 }
4990
4991 if self.state.foreign_checkout.is_some() {
4992 let wait = repository_wait(&self.state, None);
4995 let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
4996 let merged = git::git_raw(
4997 &winner.worktree,
4998 &[
4999 "-c",
5000 "core.hooksPath=/dev/null",
5001 "merge",
5002 "--no-ff",
5003 "--no-autostash",
5004 "--no-edit",
5005 "-m",
5006 &format!("Merge {tracking} before verifying"),
5007 &tip,
5008 ],
5009 )
5010 .await?;
5011 let conflict = if merged.ok() {
5012 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5013 self.state.event(
5014 "land",
5015 format!("merged {tracking} into {} before verifying", winner.branch),
5016 );
5017 None
5018 } else {
5019 if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5020 git::git(&winner.worktree, &["merge", "--abort"]).await?;
5021 }
5022 let why = format!(
5023 "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5024 winner.branch,
5025 merged.stderr.chars().take(600).collect::<String>()
5026 );
5027 self.state.status = RunStatus::Blocked;
5028 self.state.event("land", &why);
5029 Some(why)
5030 };
5031 let synced = conflict.is_none();
5032 self.state.base_sync = Some(BaseSync {
5033 tip,
5034 behind: if synced { 0 } else { behind },
5035 attempts: attempts + 1,
5036 conflict,
5037 already_in: None,
5038 });
5039 self.state.save()?;
5040 return Ok(());
5041 }
5042
5043 self.state.event(
5044 "land",
5045 format!(
5046 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5047 winner.branch
5048 ),
5049 );
5050 self.state.save()?;
5051
5052 let branch_tracking = format!("{remote}/{}", winner.branch);
5058 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5059 let remote_tip = if self.state.task_pr_branch().is_none()
5060 && matches!(&fetched_branch, Ok(o) if o.ok())
5061 {
5062 git::rev_parse(&repo, &branch_tracking).await.ok()
5063 } else {
5064 None
5065 };
5066 if let Some(theirs) = &remote_tip
5070 && !git::is_ancestor(&repo, theirs, &head).await
5071 && !crate::reconcile::origin_missing(&repo, &head, theirs)
5072 .await
5073 .is_ok_and(|missing| missing.is_empty())
5074 {
5075 let why = format!(
5076 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5077 them",
5078 short(theirs),
5079 winner.branch
5080 );
5081 self.state.status = RunStatus::Blocked;
5082 self.state.base_sync = Some(BaseSync {
5083 tip,
5084 behind,
5085 attempts: attempts + usize::from(recovery),
5086 conflict: Some(why.clone()),
5087 already_in: None,
5088 });
5089 self.state.event("land", why);
5090 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5091 .await?;
5092 self.state.save()?;
5093 return Ok(());
5094 }
5095
5096 let scratch = self.state.dir().join("base-sync");
5097 let rebased =
5098 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5099 .await
5100 {
5101 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5102 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5103 Err(e) => Err(e),
5104 };
5105 let attempts = attempts + 1;
5106 let stop_kind = match &rebased {
5107 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5108 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5109 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5110 };
5111 match rebased {
5112 Ok(None) => {
5113 git::sync_to_head(&winner.worktree).await?;
5117 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5118 let mut conflict = None;
5119 if let Some(pinned) = &remote_tip {
5120 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5121 match pushed {
5122 Ok(o) if o.ok() => self.state.event(
5123 "land",
5124 format!("pushed rebased {} to {remote}", winner.branch),
5125 ),
5126 Ok(o) => {
5127 conflict = Some(format!(
5128 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5129 winner.branch,
5130 short(pinned),
5131 o.stderr.chars().take(600).collect::<String>()
5132 ));
5133 }
5134 Err(e) => {
5135 conflict = Some(format!(
5136 "rebased {} locally but could not push it: {e:#}",
5137 winner.branch
5138 ));
5139 }
5140 }
5141 }
5142 if let Some(why) = &conflict {
5143 self.state.status = RunStatus::Blocked;
5144 self.state.event("land", why.clone());
5145 }
5146 self.state.base_sync = Some(BaseSync {
5147 tip: tip.clone(),
5148 behind: 0,
5149 attempts,
5150 conflict,
5151 already_in: None,
5152 });
5153 self.state
5154 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5155 }
5156 Ok(Some(conflict)) => {
5157 let why = format!(
5158 "{} conflicts with {tracking} and did not rebase: {}",
5159 winner.branch,
5160 conflict.chars().take(600).collect::<String>()
5161 );
5162 self.state.status = RunStatus::Blocked;
5163 self.state.base_sync = Some(BaseSync {
5164 tip,
5165 behind,
5166 attempts,
5167 conflict: Some(why.clone()),
5168 already_in: None,
5169 });
5170 self.state.event("land", why);
5171 }
5172 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5176 self.state.save()?;
5177 return Ok(());
5178 }
5179 Err(e) => {
5180 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5181 self.state.status = RunStatus::Blocked;
5182 self.state.base_sync = Some(BaseSync {
5183 tip,
5184 behind,
5185 attempts,
5186 conflict: Some(why.clone()),
5187 already_in: None,
5188 });
5189 self.state.event("land", why);
5190 }
5191 }
5192 if !self.state.base_sync_blocked() {
5193 self.state.base_sync_recovery.stop = None;
5194 }
5195 self.hold_base_sync_kind(stop_kind).await?;
5196 self.state.save()?;
5197 Ok(())
5198 }
5199
5200 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5206 if self.state.task_pr_branch().is_some() {
5209 return None;
5210 }
5211 let repo = self.state.repo.clone();
5212 let remote = self.state.config.merge.remote.clone();
5213 let fetched = git::fetch(&repo, &remote, branch).await;
5214 if !matches!(&fetched, Ok(o) if o.ok()) {
5215 return None;
5216 }
5217 let branch_tracking = format!("{remote}/{branch}");
5218 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5219 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5220 return None;
5221 }
5222 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5223 .await
5224 .is_ok_and(|missing| missing.is_empty())
5225 {
5226 return Some(format!(
5227 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5228 them",
5229 short(&theirs)
5230 ));
5231 }
5232 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5233 Ok(o) if o.ok() => {
5234 self.state
5235 .event("land", format!("pushed rebased {branch} to {remote}"));
5236 None
5237 }
5238 Ok(o) => Some(format!(
5239 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5240 someone may have pushed): {}",
5241 short(&theirs),
5242 o.stderr.chars().take(600).collect::<String>()
5243 )),
5244 Err(e) => Some(format!(
5245 "{branch} is rebased locally but could not be pushed: {e:#}"
5246 )),
5247 }
5248 }
5249
5250 async fn settle_already_in(
5267 &mut self,
5268 branch: &str,
5269 tip: &str,
5270 head: &str,
5271 attempts: usize,
5272 behind: usize,
5273 ) -> Result<bool> {
5274 let repo = self.state.repo.clone();
5275 let remote = self.state.config.merge.remote.clone();
5276 let start = self.state.base_commit.clone();
5277 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5278 Ok(Some(e)) => e,
5279 Ok(None) => return Ok(false),
5280 Err(e) => {
5281 tracing::warn!("already-in-base check for {branch}: {e:#}");
5282 return Ok(false);
5283 }
5284 };
5285 let mut verified = vec![head.to_owned()];
5286 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5287 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5288 if matches!(&fetched, Ok(o) if o.ok())
5289 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5290 && theirs != head
5291 {
5292 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5293 Ok(Some(_)) => verified.push(theirs),
5294 _ => return Ok(false),
5295 }
5296 }
5297 let base_branch = self.state.base_branch.clone();
5298 let message = format!(
5299 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5300 land",
5301 evidence.names(),
5302 evidence.proof.as_str()
5303 );
5304 let closed =
5305 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5306 .await;
5307 match closed {
5308 Ok(Ok(url)) => self
5309 .state
5310 .event("land", format!("closed {url}: superseded on {base_branch}")),
5311 Ok(Err(why)) => self
5312 .state
5313 .event("land", format!("did not close a pull request: {why}")),
5314 Err(e) => {
5315 let why = format!(
5316 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5317 not be closed ({e:#}); resume to retry"
5318 );
5319 self.state.status = RunStatus::Blocked;
5320 self.state.base_sync = Some(BaseSync {
5321 tip: tip.to_owned(),
5322 behind,
5323 attempts,
5324 conflict: Some(why.clone()),
5325 already_in: None,
5326 });
5327 self.state.event("land", why);
5328 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5329 .await?;
5330 self.state.save()?;
5331 return Ok(true);
5332 }
5333 }
5334 self.state.status = RunStatus::AlreadyInBase;
5335 self.state.base_sync_recovery.stop = None;
5336 self.state.base_sync = Some(BaseSync {
5337 tip: tip.to_owned(),
5338 behind,
5339 attempts,
5340 conflict: None,
5341 already_in: Some(evidence),
5342 });
5343 self.state.event("land", message);
5344 self.state.save()?;
5345 self.settle_questions();
5346 Ok(true)
5347 }
5348
5349 fn landing_base(&self) -> String {
5359 self.state
5360 .base_sync
5361 .as_ref()
5362 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5363 }
5364
5365 pub async fn fix_selected(
5398 &mut self,
5399 ids: &[String],
5400 reason: &str,
5401 allow_stale: bool,
5402 ) -> Result<()> {
5403 let reason = reason.trim();
5404 if reason.is_empty() {
5405 bail!("a fix request needs a reason — that is the operator's own record of why");
5406 }
5407 if ids.is_empty() {
5408 bail!("no finding id given");
5409 }
5410 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5411 bail!(
5412 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5413 has already concluded — can be given a targeted fix. A run still \
5414 in progress should simply be resumed; a `merged` run's branch has \
5415 already landed, so its answer is a fresh `magi review <branch>`, \
5416 not reopening this run's own record",
5417 self.state.id,
5418 self.state.status.as_str()
5419 );
5420 }
5421 let Some(winner) = self.state.winner().cloned() else {
5422 bail!("run {} has no winning candidate to fix", self.state.id);
5423 };
5424 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5425 bail!(
5426 "branch `{}` no longer exists; this run cannot be extended",
5427 winner.branch
5428 );
5429 }
5430 let home = crate::run::home();
5431 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5432 bail!(
5433 "run {} is currently being worked on by another magi process",
5434 self.state.id
5435 );
5436 }
5437 let _claim = FixClaim::acquire(&self.state.dir())?;
5443
5444 let mut seen = BTreeSet::new();
5448 let mut findings = Vec::new();
5449 let mut missing = Vec::new();
5450 for id in ids {
5451 if !seen.insert(id.clone()) {
5452 continue;
5453 }
5454 match self.state.finding(id) {
5455 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5456 id: f.id.clone(),
5457 severity: f.severity,
5458 reviewer_vote: rec.vote,
5459 round: round.round,
5460 round_head: round.head.clone(),
5461 reviewer: rec.reviewer,
5462 agent: rec.agent.clone(),
5463 file: f.file.clone(),
5464 line: f.line,
5465 title: f.title.clone(),
5466 detail: f.detail.clone(),
5467 outcome: OperatorFixOutcome::Pending,
5468 }),
5469 None => missing.push(id.clone()),
5470 }
5471 }
5472 if !missing.is_empty() {
5473 bail!(
5474 "unknown finding id(s): {}; nothing was changed",
5475 missing.join(", ")
5476 );
5477 }
5478
5479 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5480 let stale_details: Vec<(String, String)> = findings
5481 .iter()
5482 .filter(|f| f.round_head != head_at_request)
5483 .map(|f| (f.id.clone(), f.round_head.clone()))
5484 .collect();
5485 let stale = !stale_details.is_empty();
5486 if stale && !allow_stale {
5487 bail!(
5488 "the branch has moved since some finding(s) were raised — {} — now \
5489 at {}; pass --allow-stale to fix anyway, or re-run review first",
5490 stale_details
5491 .iter()
5492 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5493 .collect::<Vec<_>>()
5494 .join(", "),
5495 short(&head_at_request)
5496 );
5497 }
5498
5499 let request = OperatorFixRequest {
5500 requested_at: Timestamp::now(),
5501 reason: reason.to_owned(),
5502 findings,
5503 head_at_request: head_at_request.clone(),
5504 allow_stale,
5505 stale,
5506 fix: None,
5507 result_head: None,
5508 follow_up_review_run: None,
5509 };
5510 self.state.event(
5511 "fix",
5512 format!(
5513 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5514 request.findings.len(),
5515 request
5516 .findings
5517 .iter()
5518 .map(|f| f.id.as_str())
5519 .collect::<Vec<_>>()
5520 .join(", "),
5521 ),
5522 );
5523 self.state.operator_fixes.push(request);
5530 self.state.save()?;
5531 let request_index = self.state.operator_fixes.len() - 1;
5532
5533 let repo = self.state.repo.clone();
5542 let coordination = self.coordinate(&repo).await?;
5543 if winner.worktree.exists() {
5544 let dirty = git::git(
5547 &winner.worktree,
5548 &["status", "--porcelain", "--untracked-files=all"],
5549 )
5550 .await?;
5551 let only_withheld = dirty.lines().all(|l| {
5552 l.strip_prefix("?? ")
5553 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5554 });
5555 if !only_withheld {
5556 bail!(
5557 "`{}` has uncommitted changes; refusing to touch it — commit or \
5558 discard them first",
5559 winner.worktree.display()
5560 );
5561 }
5562 git::worktree_remove(&self.state.repo, &winner.worktree)
5563 .await
5564 .ok();
5565 }
5566 let fix_worktree = self.state.worktree_root().join("operator-fix");
5567 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5568 git::git(
5569 &self.state.repo,
5570 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5571 )
5572 .await
5573 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5574 if !git::is_clean(&fix_worktree).await? {
5575 git::worktree_remove(&self.state.repo, &fix_worktree)
5576 .await
5577 .ok();
5578 bail!(
5579 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5580 winner.branch
5581 );
5582 }
5583 let repo_for_setup = self.state.repo.clone();
5584 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5585 Ok(()) => {
5586 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5587 }
5588 Err(e) => Err(e),
5589 };
5590 if let Err(e) = setup {
5591 git::worktree_remove(&self.state.repo, &fix_worktree)
5592 .await
5593 .ok();
5594 return Err(e);
5595 }
5596
5597 let run_id = self.state.id.clone();
5598 let prompts = self.state.config.prompts.clone();
5599 let language = self.state.config.graph.language.clone();
5600 let sessions = self.state.config.graph.sessions;
5601 let artifacts = agent::artifacts_dir(&self.state.dir());
5602 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5603 .findings
5604 .iter()
5605 .map(|f| Finding {
5606 id: f.id.clone(),
5607 severity: f.severity,
5608 file: f.file.clone(),
5609 line: f.line,
5610 title: f.title.clone(),
5611 detail: f.detail.clone(),
5612 })
5613 .collect();
5614 let fix_prompt = prompt::operator_fix_in(
5615 &self.state.instruction,
5616 &finding_list,
5617 reason,
5618 &stale_details,
5619 &head_at_request,
5620 &language,
5621 &self.state.config.graph.github_language,
5622 );
5623 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5624 drop(coordination);
5625 let (job, seat, out) = self
5626 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5627 prompt: fix_prompt.clone(),
5628 spec,
5629 seat,
5630 cwd: fix_worktree.clone(),
5631 timeout,
5632 allow_write: true,
5633 sessions,
5634 artifacts: artifacts.clone(),
5635 stem: "operator-fix".to_owned(),
5636 handover: None,
5637 })
5638 .await;
5639 let agent_id = seat.agent.clone();
5640
5641 let mut fix = FixRecord {
5642 agent: agent_id,
5643 addressed: Vec::new(),
5644 rejected: Vec::new(),
5645 notes: String::new(),
5646 committed: false,
5647 failed: None,
5648 duration_ms: 0,
5649 continuation: None,
5650 };
5651 let mut final_seat = seat.clone();
5652 match out {
5653 AgentOutcome::Ok(o) => {
5654 fix.duration_ms = o.duration_ms;
5655 let parsed = verdict::extract_json::<FixReport>(&o.text);
5656 let incomplete_reason = match &parsed {
5657 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5658 "the reply parsed, but it reported a command whose own CLI \
5659 never confirmed an exit status"
5660 .to_owned(),
5661 ),
5662 Ok(_) => None,
5663 Err(e) => Some(e.to_string()),
5664 };
5665 match incomplete_reason {
5666 None => {
5667 let report = parsed.expect("checked Ok above");
5668 fix.addressed = report.addressed;
5669 fix.rejected = report.rejected;
5670 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5671 }
5672 Some(reason) => {
5673 let (resumed_seat, resolved, failure, cont) = self
5674 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5675 .await;
5676 fix.duration_ms += cont.cumulative_wait_ms;
5677 fix.continuation = Some(cont);
5678 final_seat = resumed_seat;
5679 match resolved {
5680 Some(report) => {
5681 fix.addressed = report.addressed;
5682 fix.rejected = report.rejected;
5683 fix.notes =
5684 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5685 }
5686 None => fix.failed = failure,
5687 }
5688 }
5689 }
5690 }
5691 AgentOutcome::Dropped(o) => {
5692 fix.duration_ms = o.duration_ms;
5693 let why = o
5694 .dropped
5695 .as_ref()
5696 .map(|d| d.why.as_str())
5697 .unwrap_or("the CLI ended the stream without delivering its answer");
5698 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5699 }
5700 AgentOutcome::Quota(o) => {
5701 self.state.quota.push(QuotaLoss {
5702 seat: final_seat.key.clone(),
5703 node: "fix".to_owned(),
5704 at: Timestamp::now(),
5705 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5706 });
5707 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5708 }
5709 AgentOutcome::Failed(e) => fix.failed = Some(e),
5710 AgentOutcome::ResourceBlocked(e) => {
5711 fix.failed = Some(format!("the fixer was not started: {e}"));
5712 }
5713 }
5714 if fix.continuation.is_none() {
5715 fix.continuation = Some(ContinuationRecord::not_needed());
5716 }
5717 self.state.seats.insert(final_seat.key.clone(), final_seat);
5718
5719 let rescue_message = format!(
5720 "magi: operator-selected fix ({}) (uncommitted work)",
5721 self.state.operator_fixes[request_index]
5722 .findings
5723 .iter()
5724 .map(|f| f.id.as_str())
5725 .collect::<Vec<_>>()
5726 .join(", ")
5727 );
5728 let repo = self.state.repo.clone();
5729 let coordination = self.coordinate(&repo).await?;
5730 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5731 self.state.note_withheld("fix", &r.withheld);
5732 }
5733 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5734 fix.committed = after != head_at_request;
5735 git::worktree_remove(&self.state.repo, &fix_worktree)
5736 .await
5737 .ok();
5738 drop(coordination);
5739
5740 self.state.event(
5741 "fix",
5742 match &fix.failed {
5743 Some(reason) => format!(
5744 "operator fix: adoption report was lost ({reason}); {}",
5745 if fix.committed {
5746 "committed"
5747 } else {
5748 "NO new commit"
5749 }
5750 ),
5751 None => format!(
5752 "operator fix: {} addressed, {} rejected, {}",
5753 fix.addressed.len(),
5754 fix.rejected.len(),
5755 if fix.committed {
5756 "committed"
5757 } else {
5758 "NO new commit"
5759 }
5760 ),
5761 },
5762 );
5763
5764 for f in &mut self.state.operator_fixes[request_index].findings {
5771 f.outcome = if fix.failed.is_some() {
5772 OperatorFixOutcome::Unreported
5773 } else if fix.addressed.contains(&f.id) {
5774 OperatorFixOutcome::Addressed
5775 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5776 OperatorFixOutcome::Rejected { why: r.why.clone() }
5777 } else {
5778 OperatorFixOutcome::Unreported
5779 };
5780 }
5781
5782 let committed = fix.committed;
5783 if committed {
5784 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5785 }
5786 self.state.operator_fixes[request_index].fix = Some(fix);
5787 self.state.save()?;
5790
5791 if committed {
5792 self.state.event(
5793 "fix",
5794 format!(
5795 "operator fix committed {}; opening a follow-up review-only run",
5796 short(&after)
5797 ),
5798 );
5799 let origin =
5802 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5803 match Self::review(
5804 &self.state.repo,
5805 &winner.branch,
5806 self.state.config.clone(),
5807 origin,
5808 )
5809 .await
5810 {
5811 Ok(mut follow_up) => {
5812 follow_up.state.event(
5813 "start",
5814 format!(
5815 "requested by an operator fix on run {} for finding(s) {}",
5816 self.state.id,
5817 self.state.operator_fixes[request_index]
5818 .findings
5819 .iter()
5820 .map(|f| f.id.as_str())
5821 .collect::<Vec<_>>()
5822 .join(", "),
5823 ),
5824 );
5825 follow_up.state.save()?;
5826 let follow_up_id = follow_up.state.id.clone();
5827 let adopted = match crate::direct::adopt(
5830 &follow_up.state,
5831 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5832 ) {
5833 Ok(a) => a,
5834 Err(e) => {
5835 self.state.event(
5838 "fix",
5839 format!(
5840 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5841 ),
5842 );
5843 self.state.save()?;
5844 return Ok(());
5845 }
5846 };
5847 let executed = follow_up.execute().await;
5848 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5849 if let Some(a) = adopted {
5850 a.finish(&follow_up.state, executed);
5851 }
5852 if let Some(e) = failure {
5853 self.state.event(
5854 "fix",
5855 format!(
5856 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5857 ),
5858 );
5859 }
5860 self.state.operator_fixes[request_index].follow_up_review_run =
5861 Some(follow_up_id);
5862 }
5863 Err(e) => {
5864 self.state.event(
5865 "fix",
5866 format!("committed the fix but could not open a follow-up review: {e:#}"),
5867 );
5868 }
5869 }
5870 self.state.save()?;
5871 }
5872
5873 Ok(())
5874 }
5875
5876 async fn ask_fixer(
5896 &mut self,
5897 winner: &Candidate,
5898 node: &'static str,
5899 round: Option<usize>,
5900 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5901 ) -> (SeatJob, SeatState, AgentOutcome) {
5902 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5903 self.ask_fixer_among(attempts, node, round, None, build)
5904 .await
5905 }
5906
5907 async fn ask_fixer_among(
5912 &mut self,
5913 attempts: Vec<(AgentSpec, String)>,
5914 node: &'static str,
5915 round: Option<usize>,
5916 restore: Option<(&Path, &str)>,
5917 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5918 ) -> (SeatJob, SeatState, AgentOutcome) {
5919 let run_id = self.state.id.clone();
5920 let prompts = self.state.config.prompts.clone();
5921 let cache = self.state.config.cache_dir();
5922 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5923 let mut last = None;
5924 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5925 let seat = self.seat(&key, &spec.id);
5926 let mut job = build(spec.clone(), seat);
5927 if i > 0 {
5928 job.stem = format!("{}-{}", job.stem, spec.id);
5929 }
5930 let ctx = WaveCtx {
5931 carry_seats: false,
5932 pauses: Some((&self.pause, &self.interrupt)),
5933 run: &run_id,
5934 node,
5935 prompts: &prompts,
5936 cache: cache.as_deref(),
5937 round,
5938 };
5939 let (seat, out) =
5940 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5941 let advances = match &out {
5944 AgentOutcome::Ok(o) => agent::output_advances(o),
5945 AgentOutcome::ResourceBlocked(_) => false,
5946 _ => true,
5947 };
5948 if let Some(next) = ids.get(i + 1)
5949 && advances
5950 {
5951 self.state.seats.insert(seat.key.clone(), seat.clone());
5952 let class =
5953 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5954 let reason = match &out {
5955 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5956 other => fail_reason(other),
5957 };
5958 if let Some((wt, rev)) = restore
5962 && let Err(e) = self.graft_restore(wt, rev).await
5963 {
5964 let why =
5965 format!("could not restore the worktree before the next fixer: {e:#}");
5966 tracing::warn!("{why}");
5967 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5968 AgentOutcome::ResourceBlocked(why)
5969 } else {
5970 AgentOutcome::Failed(why)
5971 };
5972 last = Some((job, seat, out));
5973 break;
5974 }
5975 record_handover(
5976 &mut self.state,
5977 node,
5978 &seat.key,
5979 &spec.id,
5980 next,
5981 &class,
5982 &reason,
5983 );
5984 continue;
5985 }
5986 last = Some((job, seat, out));
5987 break;
5988 }
5989 last.expect("the fixer chain always has an entry")
5990 }
5991
5992 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6003 use crate::recovery::{Mode, Repair};
6004 if !self.state.reviews.is_empty() {
6005 return Ok(());
6006 }
6007 let Some(mut rec) = self.state.recovery.clone() else {
6008 return Ok(());
6009 };
6010 if rec.mode != Mode::VerifyRepair {
6011 return Ok(());
6012 }
6013 let wt = winner.worktree.clone();
6014 let mut head = git::rev_parse(&wt, "HEAD").await?;
6015 match rec.repair.clone() {
6016 Repair::Done { .. } => return Ok(()),
6017 Repair::WaitingForRestore { before } => {
6018 self.graft_restore(&wt, &before).await?;
6019 head = before;
6020 rec.repair = Repair::NotStarted;
6021 self.state.recovery = Some(rec.clone());
6022 self.state.save()?;
6023 }
6024 Repair::Started { before } => {
6025 if let Err(e) = self.graft_restore(&wt, &before).await {
6029 if e.is::<crate::repo_lock::ResourceBlocked>() {
6030 return Err(e);
6031 }
6032 return self
6033 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6034 .await;
6035 }
6036 rec.repair = Repair::Done {
6037 before: before.clone(),
6038 after: before,
6039 committed: false,
6040 failed: Some("interrupted; rolled back".to_owned()),
6041 };
6042 self.state
6043 .event("recovery", "the repair was interrupted and rolled back");
6044 self.state.recovery = Some(rec);
6045 self.state.save()?;
6046 return Ok(());
6047 }
6048 Repair::NotStarted => {}
6049 }
6050 if crate::recovery::recheck(&mut rec, &head) {
6051 self.state.event("recovery", rec.summary());
6052 self.state.recovery = Some(rec);
6053 self.state.save()?;
6054 return Ok(());
6055 }
6056 rec.repair = Repair::Started {
6057 before: head.clone(),
6058 };
6059 self.state.recovery = Some(rec.clone());
6060 self.state.save()?;
6061
6062 let language = self.state.config.graph.language.clone();
6063 let sessions = self.state.config.graph.sessions;
6064 let artifacts = agent::artifacts_dir(&self.state.dir());
6065 let fix_prompt = prompt::recovery_fix(
6066 &self.state.instruction,
6067 &rec,
6068 self.state.config.graph.review_rounds,
6069 &language,
6070 );
6071 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6072 self.state.event(
6073 "recovery",
6074 format!(
6075 "repair: the fixer starts from the failed verification on {}",
6076 short(&head)
6077 ),
6078 );
6079 let (_job, seat, out) = self
6080 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6081 prompt: fix_prompt.clone(),
6082 spec,
6083 seat,
6084 cwd: wt.clone(),
6085 timeout,
6086 allow_write: true,
6087 sessions,
6088 artifacts: artifacts.clone(),
6089 stem: "fix-recovery".to_owned(),
6090 handover: None,
6091 })
6092 .await;
6093 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6094 rec.repair = Repair::WaitingForRestore { before: head };
6095 self.state.recovery = Some(rec);
6096 return self.stop_for_resource();
6097 }
6098 let failed = match &out {
6099 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6100 AgentOutcome::Quota(o) => {
6101 self.state.quota.push(QuotaLoss {
6102 seat: seat.key.clone(),
6103 node: "fix".to_owned(),
6104 at: Timestamp::now(),
6105 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6106 });
6107 Some("rate limited (quota); the fixer could not run".to_owned())
6108 }
6109 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6110 AgentOutcome::Failed(e) => Some(e.clone()),
6111 };
6112 self.state.seats.insert(seat.key.clone(), seat);
6113 let coordination = self.coordinate(&wt).await?;
6114 let failed = match failed {
6115 Some(why) => {
6117 if let Err(e) = git::restore_to(&wt, &head).await {
6118 return self
6119 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6120 .await;
6121 }
6122 Some(why)
6123 }
6124 None => {
6125 if let Ok(r) = git::rescue_commit(
6126 &wt,
6127 "magi: recovery repair of the failed verification (uncommitted work)",
6128 )
6129 .await
6130 {
6131 self.state.note_withheld("fix", &r.withheld);
6132 }
6133 None
6134 }
6135 };
6136 let after = git::rev_parse(&wt, "HEAD").await?;
6137 drop(coordination);
6138 self.state.event(
6139 "recovery",
6140 match &failed {
6141 Some(why) => format!("repair: nothing kept ({why})"),
6142 None if after != head => {
6143 format!("repair: committed {}", short(&after))
6144 }
6145 None => "repair: the fixer made no commit".to_owned(),
6146 },
6147 );
6148 rec.repair = Repair::Done {
6149 before: head.clone(),
6150 committed: after != head,
6151 after,
6152 failed,
6153 };
6154 self.state.recovery = Some(rec);
6155 self.state.save()?;
6156 Ok(())
6157 }
6158
6159 async fn stop_recovery(
6162 &mut self,
6163 mut rec: crate::recovery::RecoveryRecord,
6164 head: &str,
6165 why: String,
6166 ) -> Result<()> {
6167 rec.repair = crate::recovery::Repair::Started {
6169 before: head.to_owned(),
6170 };
6171 self.state.recovery = Some(rec);
6172 self.state
6173 .event("recovery", format!("repair could not be undone: {why}"));
6174 self.state.status = RunStatus::Blocked;
6175 self.state.save()?;
6176 bail!("recovery repair could not be rolled back: {why}")
6177 }
6178
6179 async fn review_loop(&mut self) -> Result<()> {
6180 if self.hold_base_sync().await? {
6181 return Ok(());
6182 }
6183 loop {
6184 if self
6185 .state
6186 .base_sync
6187 .as_ref()
6188 .is_some_and(|s| s.conflict.is_some())
6189 {
6190 return Ok(());
6191 }
6192 if self.catchup_repairable() {
6193 let record = self.state.reviews.last().expect("repairable round").clone();
6194 let winner = self
6195 .state
6196 .winner()
6197 .cloned()
6198 .context("catch-up repair has no winner")?;
6199 if !self.fix_review_round(&winner, record).await? {
6200 return Ok(());
6201 }
6202 }
6203 self.review_loop_pass().await?;
6204 if !self.catchup_repairable() {
6205 return Ok(());
6206 }
6207 }
6208 }
6209
6210 fn catchup_repairable(&self) -> bool {
6213 self.state.reviews.len() < self.state.config.graph.review_rounds
6214 && self.state.reviews.last().is_some_and(|r| {
6215 !r.clean
6216 && r.blocking == 0
6217 && r.fix.is_none()
6218 && r.e2e_status() == E2eStatus::Failed
6219 })
6220 }
6221
6222 async fn review_loop_pass(&mut self) -> Result<()> {
6223 if self
6228 .state
6229 .base_sync
6230 .as_ref()
6231 .is_some_and(|s| s.conflict.is_some())
6232 {
6233 return Ok(());
6234 }
6235 let run_id = self.state.id.clone();
6240 let prompts = self.state.config.prompts.clone();
6241 let Some(winner) = self.state.winner().cloned() else {
6242 return Ok(());
6243 };
6244 let max_rounds = self.state.config.graph.review_rounds;
6245 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6255 if status == RunStatus::Gating {
6258 self.record_contested_handoff();
6259 }
6260 self.state.status = status;
6261 self.state.save()?;
6262 return Ok(());
6263 }
6264 self.state.status = RunStatus::Reviewing;
6265 if self.state.reviews.last().is_some_and(|r| {
6275 r.e2e_status() == E2eStatus::ResourceBlocked
6276 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6277 }) {
6278 let shell = self.state.config.shell();
6279 return self
6280 .stop_unresolved_review(
6281 "the last round's own verification never resolved",
6282 &shell,
6283 &winner.worktree,
6284 )
6285 .await;
6286 }
6287
6288 self.recovery_repair(&winner).await?;
6289 if self.resource_halt()? {
6290 return Ok(());
6291 }
6292
6293 let repo = self.state.repo.clone();
6294 let root = self.state.worktree_root();
6295 let language = self.state.config.graph.language.clone();
6296 let sessions = self.state.config.graph.sessions;
6297 let artifacts = self.merge_repair_artifacts();
6298 let base = self.landing_base();
6299 let base_short = short(&base);
6300 let reviewers = self.roles.reviewers.clone();
6301 let shell = self.state.config.shell();
6302
6303 for round in (self.state.reviews.len() + 1)..=max_rounds {
6304 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6305 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6306 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6307 let prev_verification = self
6316 .state
6317 .reviews
6318 .last()
6319 .and_then(|r| r.verification_summary(&head));
6320 let prior_section = match (&self.state.recovery, round) {
6323 (Some(rec), 1) if self.state.reviews.is_empty() => {
6324 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6325 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6326 Some(crate::recovery::head_of(
6327 &git::diff(&winner.worktree, reviewed, "HEAD")
6328 .await
6329 .unwrap_or_default(),
6330 crate::recovery::SINCE_DIFF_MAX,
6331 ))
6332 }
6333 _ => None,
6334 };
6335 Some(prompt::recovery_review(rec, since.as_deref()))
6336 }
6337 _ => None,
6338 };
6339
6340 let mut jobs = Vec::new();
6344 let coordination = self.coordinate(&repo).await?;
6345 for (r, spec) in reviewers.iter().cloned().enumerate() {
6346 let wt = root.join(format!("review-{}", r + 1));
6347 if wt.exists() {
6350 git::reset_detached(&wt, &head).await?;
6351 } else {
6352 git::worktree_add_detached(&repo, &wt, &head).await?;
6353 }
6354 self.setup_seat_worktree(&repo, &wt, false).await?;
6355 let seat_key = format!("review-{}", r + 1);
6356 let spec = pick_start_spec(
6359 &self.roles.reviewer_roster,
6360 spec,
6361 self.state.seat_history.get(&seat_key),
6362 );
6363 let seat = self.seat(&seat_key, &spec.id);
6364 jobs.push(SeatJob {
6365 prompt: prompt::review(&prompt::ReviewCtx {
6366 instruction: &self.state.instruction,
6367 branch: &winner.branch,
6368 base_short: &base_short,
6369 stat: &stat,
6370 patch: &patch,
6371 verification: prev_verification.as_ref(),
6372 prior: prior_section.as_deref(),
6373 reviewers: reviewers.len(),
6374 round,
6375 rounds: max_rounds,
6376 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6379 lens: Lens::for_seat(r),
6380 language: &language,
6381 github_language: &self.state.config.graph.github_language,
6382 }),
6383 spec,
6384 seat,
6385 cwd: wt,
6386 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6387 allow_write: false,
6388 sessions,
6389 artifacts: artifacts.clone(),
6390 stem: format!("review-{round}-{}", r + 1),
6391 handover: None,
6392 });
6393 }
6394
6395 drop(coordination);
6396 self.state.event(
6397 "review",
6398 format!(
6399 "round {round}: {} reviewers on {}",
6400 jobs.len(),
6401 short(&head)
6402 ),
6403 );
6404 let mut quota_losses = Vec::new();
6405 let review_retries = self.state.config.graph.retries;
6406 let review_cache = self.state.config.cache_dir();
6407 let ctx = WaveCtx {
6408 carry_seats: true,
6409 pauses: Some((&self.pause, &self.interrupt)),
6410 run: &run_id,
6411 node: "review",
6412 prompts: &prompts,
6413 cache: review_cache.as_deref(),
6414 round: Some(round),
6415 };
6416 let results = ask_json_wave::<Review>(
6417 jobs,
6418 Arc::clone(&self.sem),
6419 review_retries,
6420 &self.roles.reviewer_roster,
6421 &ctx,
6422 &mut quota_losses,
6423 &mut self.state,
6424 &|_: &Review| Ok(()),
6425 )
6426 .await;
6427 if self.state.resource_block.is_some() {
6428 return self.stop_for_resource();
6429 }
6430 let round_quota_missing = quota_losses.len();
6434 self.state.quota.extend(quota_losses);
6435
6436 let mut records = Vec::new();
6437 let mut all_findings = Vec::new();
6438 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6439 let agent_id = seat.agent.clone();
6440 self.state.seats.insert(seat.key.clone(), seat);
6441 let mut record = ReviewRecord {
6442 reviewer: r + 1,
6443 agent: agent_id,
6444 summary: String::new(),
6445 findings: Vec::new(),
6446 vote: None,
6447 failed: None,
6448 duration_ms: 0,
6449 attempts,
6455 };
6456 match res {
6457 Ok((review, out)) => {
6458 record.summary =
6466 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6467 record.vote = Some(review.vote);
6468 record.duration_ms = out.duration_ms;
6469 for (n, mut f) in review.findings.into_iter().enumerate() {
6470 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6473 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6474 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6475 f.file = f
6481 .file
6482 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6483 all_findings.push(f.clone());
6484 record.findings.push(f);
6485 }
6486 self.state.event(
6487 "review",
6488 format!(
6489 "round {round}: reviewer {} voted {} with {} finding(s)",
6490 r + 1,
6491 review.vote.label(),
6492 record.findings.len()
6493 ),
6494 );
6495 }
6496 Err(e) => {
6497 record.failed = Some(e.to_string());
6498 self.state.event(
6499 "review",
6500 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6501 );
6502 }
6503 }
6504 records.push(record);
6505 }
6506
6507 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6514 let vote_split =
6515 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6516 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6517 if vote_split {
6518 self.state.event(
6519 "review",
6520 format!(
6521 "round {round}: votes split ({}) — one round of reconsideration",
6522 initial_votes
6523 .iter()
6524 .map(|v| v.label())
6525 .collect::<Vec<_>>()
6526 .join(", ")
6527 ),
6528 );
6529 let panel: Vec<ReviewSeatReport<'_>> = records
6532 .iter()
6533 .filter_map(|r| {
6534 r.vote.map(|vote| ReviewSeatReport {
6535 reviewer: r.reviewer,
6536 vote,
6537 summary: &r.summary,
6538 findings: &r.findings,
6539 })
6540 })
6541 .collect();
6542
6543 let mut jobs = Vec::new();
6544 let mut seats_at = Vec::new();
6545 for (r, spec) in reviewers.iter().cloned().enumerate() {
6546 if records[r].vote.is_none() {
6550 continue;
6551 }
6552 let wt = root.join(format!("review-{}", r + 1));
6553 let seat_key = format!("review-{}", r + 1);
6554 let spec = self.occupant(&seat_key, spec);
6555 let seat = self.seat(&seat_key, &spec.id);
6556 let build = |with_patch: bool| {
6564 prompt::review_reconsider(&ReviewReconsiderCtx {
6565 instruction: &self.state.instruction,
6566 reviewer: r + 1,
6567 lens: Lens::for_seat(r),
6568 panel: &panel,
6569 patch: with_patch.then_some(ReviewPatch {
6570 branch: &winner.branch,
6571 base_short: &base_short,
6572 stat: &stat,
6573 patch: &patch,
6574 }),
6575 round,
6576 rounds: max_rounds,
6577 language: &language,
6578 })
6579 };
6580 let full = build(true);
6581 let prompt = if has_context(&spec, &seat, sessions) {
6582 build(false)
6583 } else {
6584 full.clone()
6585 };
6586 jobs.push(SeatJob {
6587 prompt,
6588 spec,
6589 seat,
6590 cwd: wt,
6591 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6592 allow_write: false,
6593 sessions,
6594 artifacts: artifacts.clone(),
6595 stem: format!("review-{round}-reconsider-{}", r + 1),
6596 handover: Some(full),
6597 });
6598 seats_at.push(r);
6599 }
6600
6601 let mut recon_quota_losses = Vec::new();
6602 let recon_cache = self.state.config.cache_dir();
6603 let recon_ctx = WaveCtx {
6604 carry_seats: true,
6605 pauses: Some((&self.pause, &self.interrupt)),
6606 run: &run_id,
6607 node: "review",
6608 prompts: &prompts,
6609 cache: recon_cache.as_deref(),
6610 round: Some(round),
6611 };
6612 let recon_results = ask_json_wave::<ReviewRevote>(
6613 jobs,
6614 Arc::clone(&self.sem),
6615 review_retries,
6616 &self.roles.reviewer_roster,
6617 &recon_ctx,
6618 &mut recon_quota_losses,
6619 &mut self.state,
6620 &|_: &ReviewRevote| Ok(()),
6621 )
6622 .await;
6623 if self.state.resource_block.is_some() {
6624 return self.stop_for_resource();
6625 }
6626 self.state.quota.extend(recon_quota_losses);
6627
6628 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6629 let agent_id = seat.agent.clone();
6630 self.state.seats.insert(seat.key.clone(), seat);
6631 let mut rec = ReviewRevoteRecord {
6632 reviewer: r + 1,
6633 agent: agent_id,
6634 vote: None,
6635 reason: String::new(),
6636 failed: None,
6637 };
6638 match res {
6639 Ok((rv, _)) => {
6640 rec.vote = Some(rv.vote);
6641 rec.reason =
6642 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6643 self.state.event(
6644 "review",
6645 format!(
6646 "round {round}: reviewer {} revoted {}",
6647 r + 1,
6648 rv.vote.label()
6649 ),
6650 );
6651 }
6652 Err(e) => {
6653 rec.failed = Some(e.to_string());
6654 self.state.event(
6655 "review",
6656 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6657 );
6658 }
6659 }
6660 reconsideration.push(rec);
6661 }
6662 } else if initial_votes.len() > 1 {
6663 self.state.event(
6664 "review",
6665 format!(
6666 "round {round}: votes agreed ({}) — no reconsideration",
6667 initial_votes[0].label()
6668 ),
6669 );
6670 }
6671
6672 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6673 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6674 let defer_e2e =
6685 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6686 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6687 let reason =
6688 format!("{blocking} blocking finding(s) already required a fix this round");
6689 self.state.event(
6690 "verify",
6691 format!(
6692 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6693 {}); it will run once a round has none left",
6694 short(&head)
6695 ),
6696 );
6697 (Vec::new(), false, true, Some(reason))
6698 } else {
6699 let e2e_commands = self.state.config.verify.e2e.clone();
6700 let cache_dir = self.state.config.cache_dir();
6701 let context = format!("round {round}");
6702 let wait = self.repository_wait();
6703 let (e2e, verify_retried) = with_cache_lease(
6704 &mut self.state,
6705 cache_dir.as_deref(),
6706 "e2e",
6707 "e2e",
6708 &winner.worktree,
6709 &head,
6710 verify_timeout,
6711 &context,
6712 &wait,
6713 |state, budget| {
6714 let shell = shell.clone();
6715 let e2e_commands = e2e_commands.clone();
6716 let worktree = winner.worktree.clone();
6717 let context = context.clone();
6718 async move {
6719 run_e2e_with_retry(
6720 state,
6721 &shell,
6722 &e2e_commands,
6723 &worktree,
6724 budget,
6725 &context,
6726 )
6727 .await
6728 }
6729 },
6730 )
6731 .await?;
6732 (e2e, verify_retried, false, None)
6733 };
6734
6735 let expected = records.len();
6736 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6737 let incomplete = answered < expected;
6738 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6739 let policy = self.state.config.graph.incomplete_review;
6740 let clean = round_is_clean(
6741 blocking,
6742 e2e_ok,
6743 answered,
6744 expected,
6745 round_quota_missing,
6746 policy,
6747 );
6748
6749 let mut round_record = ReviewRound {
6750 round,
6751 head: head.clone(),
6752 verified_head: None,
6753 verified_at: None,
6754 reviews: records,
6755 e2e,
6756 verify_retried,
6757 e2e_deferred,
6758 e2e_defer_reason,
6759 fix: None,
6760 blocking,
6761 answered,
6762 expected,
6763 clean,
6764 progressed: false,
6765 vote_split,
6766 reconsideration,
6767 verdict: None,
6768 };
6769 round_record.verdict = ReviewVote::worst(
6773 round_record
6774 .final_votes()
6775 .into_iter()
6776 .map(|(_, _, vote)| vote),
6777 );
6778 if !matches!(
6789 round_record.e2e_status(),
6790 E2eStatus::Deferred | E2eStatus::NotConfigured
6791 ) {
6792 round_record.verified_head = Some(head.clone());
6793 round_record.verified_at = Some(Timestamp::now());
6794 }
6795
6796 if incomplete {
6797 let missing: Vec<String> = round_record
6798 .reviews
6799 .iter()
6800 .filter(|r| r.failed.is_some())
6801 .map(|r| format!("review-{}", r.reviewer))
6802 .collect();
6803 self.state.event(
6804 "review",
6805 format!(
6806 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6807 missing.join(", ")
6808 ),
6809 );
6810 }
6811
6812 if clean {
6813 self.state.event(
6814 "review",
6815 if incomplete && policy == IncompleteReviewPolicy::Warn {
6816 format!(
6817 "round {round}: clean (warn policy, incomplete panel) — no \
6818 blocking findings from the seats that answered, verification green"
6819 )
6820 } else if incomplete {
6821 format!(
6822 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6823 quorum) — no blocking findings from the seats that answered, \
6824 verification green",
6825 expected - answered
6826 )
6827 } else {
6828 format!("round {round}: clean — no blocking findings, verification green")
6829 },
6830 );
6831 self.state.reviews.push(round_record);
6832 self.state.status = RunStatus::Gating;
6833 self.state.save()?;
6834 return Ok(());
6835 }
6836
6837 if incomplete && blocking == 0 && e2e_ok {
6845 self.state.reviews.push(round_record);
6846 self.state.save()?;
6847 if round == max_rounds {
6848 self.state.status = RunStatus::Blocked;
6849 self.state.event(
6850 "review",
6851 format!(
6852 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6853 refusing to call it clean",
6854 expected - answered
6855 ),
6856 );
6857 return Ok(());
6858 }
6859 continue;
6860 }
6861
6862 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6874 self.state.reviews.push(round_record);
6875 return self
6876 .stop_unresolved_review(
6877 "the round's own verification could not run",
6878 &shell,
6879 &winner.worktree,
6880 )
6881 .await;
6882 }
6883
6884 if round == max_rounds {
6885 self.state.reviews.push(round_record);
6886 return self
6887 .stop_reviewing(
6888 &format!(
6889 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6890 ),
6891 &shell,
6892 &winner.worktree,
6893 )
6894 .await;
6895 }
6896
6897 if !self.fix_review_round(&winner, round_record).await? {
6898 return Ok(());
6899 }
6900 }
6901 Ok(())
6902 }
6903
6904 async fn fix_review_round(
6907 &mut self,
6908 winner: &Candidate,
6909 mut round_record: ReviewRound,
6910 ) -> Result<bool> {
6911 let round = round_record.round;
6912 let shell = self.state.config.shell();
6913 let step = self.run_fix_round(winner, &mut round_record).await?;
6914 if matches!(step, FixStep::Resource) {
6915 self.stop_for_resource()?;
6916 return Ok(false);
6917 }
6918 if self.state.reviews.len() < round {
6919 self.state.reviews.push(round_record);
6920 } else {
6921 self.state.reviews[round - 1] = round_record;
6922 }
6923 self.state.save()?;
6924
6925 if matches!(step, FixStep::Recorded { report_lost: true }) {
6938 self.stop_reviewing(
6939 "the fixer's adoption report never came back, even after resuming its \
6940 own seat; refusing to start another round against the same worktree \
6941 while that is unresolved",
6942 &shell,
6943 &winner.worktree,
6944 )
6945 .await?;
6946 return Ok(false);
6947 }
6948
6949 let streak = self
6950 .state
6951 .reviews
6952 .iter()
6953 .rev()
6954 .take_while(|r| !r.progressed)
6955 .count();
6956 if streak >= STAGNANT_LIMIT {
6957 self.stop_reviewing(
6958 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6959 &shell,
6960 &winner.worktree,
6961 )
6962 .await?;
6963 return Ok(false);
6964 }
6965 Ok(true)
6966 }
6967
6968 async fn run_fix_round(
6970 &mut self,
6971 winner: &Candidate,
6972 round_record: &mut ReviewRound,
6973 ) -> Result<FixStep> {
6974 let round = round_record.round;
6975 let max_rounds = self.state.config.graph.review_rounds;
6976 let run_id = self.state.id.clone();
6977 let prompts = self.state.config.prompts.clone();
6978 let language = self.state.config.graph.language.clone();
6979 let sessions = self.state.config.graph.sessions;
6980 let artifacts = self.merge_repair_artifacts();
6981 let base = self.landing_base();
6982 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6983 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6984 let this_round_verification = round_record.verification_summary(&head);
6985 let all_findings: Vec<_> = round_record
6986 .reviews
6987 .iter()
6988 .flat_map(|r| r.findings.iter().cloned())
6989 .collect();
6990 let blocking_findings: Vec<_> = all_findings
6993 .iter()
6994 .filter(|f| f.severity.blocks())
6995 .cloned()
6996 .collect();
6997 let fix_prompt = prompt::fix_in(
6998 &self.state.instruction,
6999 &blocking_findings,
7000 this_round_verification.as_ref(),
7001 round,
7002 max_rounds,
7003 &language,
7004 &self.state.config.graph.github_language,
7005 );
7006 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7007 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7008 let (job, seat, out) = self
7009 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7010 prompt: fix_prompt.clone(),
7011 spec,
7012 seat,
7013 cwd: winner.worktree.clone(),
7014 timeout,
7015 allow_write: true,
7016 sessions,
7017 artifacts: artifacts.clone(),
7018 stem: format!("fix-{round}"),
7019 handover: None,
7020 })
7021 .await;
7022 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7025 return Ok(FixStep::Resource);
7026 }
7027 let agent_id = seat.agent.clone();
7028
7029 let mut fix = FixRecord {
7030 agent: agent_id,
7031 addressed: Vec::new(),
7032 rejected: Vec::new(),
7033 notes: String::new(),
7034 committed: false,
7035 failed: None,
7036 duration_ms: 0,
7037 continuation: None,
7038 };
7039 let mut continuation = ContinuationRecord::not_needed();
7040 let mut final_seat = seat.clone();
7041 match out {
7042 AgentOutcome::Ok(o) => {
7043 fix.duration_ms = o.duration_ms;
7044 let parsed = verdict::extract_json::<FixReport>(&o.text);
7045 let incomplete_reason = match &parsed {
7052 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7053 "the reply parsed, but it reported a command whose own CLI never \
7054 confirmed an exit status"
7055 .to_owned(),
7056 ),
7057 Ok(_) => None,
7058 Err(e) => Some(e.to_string()),
7059 };
7060 match incomplete_reason {
7061 None => {
7062 let report = parsed.expect("checked Ok above");
7063 fix.addressed = report.addressed;
7064 fix.rejected = report.rejected;
7065 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7066 }
7067 Some(reason) => {
7068 let (resumed_seat, resolved, failure, cont) = self
7069 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7070 .await;
7071 fix.duration_ms += cont.cumulative_wait_ms;
7072 continuation = cont;
7073 final_seat = resumed_seat;
7074 match resolved {
7075 Some(report) => {
7076 fix.addressed = report.addressed;
7077 fix.rejected = report.rejected;
7078 fix.notes =
7079 blind::sanitize_prose(&report.notes, &self.state.config.blind);
7080 }
7081 None => fix.failed = failure,
7082 }
7083 }
7084 }
7085 }
7086 AgentOutcome::Dropped(o) => {
7088 fix.duration_ms = o.duration_ms;
7089 let why = o
7090 .dropped
7091 .as_ref()
7092 .map(|d| d.why.as_str())
7093 .unwrap_or("the CLI ended the stream without delivering its answer");
7094 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7095 }
7096 AgentOutcome::Quota(o) => {
7097 self.state.quota.push(QuotaLoss {
7098 seat: final_seat.key.clone(),
7099 node: "fix".to_owned(),
7100 at: Timestamp::now(),
7101 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7102 });
7103 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7104 }
7105 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7106 }
7107 fix.continuation = Some(continuation);
7108 self.state.seats.insert(final_seat.key.clone(), final_seat);
7109 let coordination = self.coordinate(&winner.worktree).await?;
7110 if let Ok(r) = git::rescue_commit(
7111 &winner.worktree,
7112 &format!("magi: review round {round} fixes (uncommitted work)"),
7113 )
7114 .await
7115 {
7116 self.state.note_withheld("fix", &r.withheld);
7117 }
7118 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7119 fix.committed = after != before;
7120 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7128 drop(coordination);
7129 let progressed = diff_after != patch;
7130 let commit_note = if fix.committed {
7131 "committed"
7132 } else {
7133 "NO new commit"
7134 };
7135 let tree_note = if progressed {
7136 "changed vs base"
7137 } else {
7138 "unchanged vs base"
7139 };
7140 self.state.event(
7141 "fix",
7142 match &fix.failed {
7143 Some(reason) => {
7149 format!(
7150 "round {round}: fixer's adoption report was lost ({reason}); \
7151 {commit_note}, tree {tree_note}"
7152 )
7153 }
7154 None => format!(
7155 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7156 {tree_note}{}",
7157 fix.addressed.len(),
7158 fix.rejected.len(),
7159 if continuation.outcome == ContinuationOutcome::Resumed {
7160 format!(
7161 " (adoption report recovered after {} continuation(s))",
7162 continuation.attempts
7163 )
7164 } else {
7165 String::new()
7166 },
7167 ),
7168 },
7169 );
7170 round_record.fix = Some(fix);
7171 round_record.progressed = progressed;
7172 Ok(FixStep::Recorded {
7173 report_lost: matches!(
7174 continuation.outcome,
7175 ContinuationOutcome::Exhausted
7176 | ContinuationOutcome::QuotaLost
7177 | ContinuationOutcome::NoSession
7178 ),
7179 })
7180 }
7181
7182 fn record_contested_handoff(&mut self) {
7216 if self.state.contested_handoff.is_some() {
7217 return;
7218 }
7219 let Some(contested) = self
7220 .state
7221 .reviews
7222 .last()
7223 .and_then(ReviewRound::contested_handoff)
7224 else {
7225 return;
7226 };
7227 self.state.event(
7228 "review",
7229 format!(
7230 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7231 wait for the owner's approval",
7232 contested.findings.len(),
7233 contested.rejecters.len()
7234 ),
7235 );
7236 self.state.contested_handoff = Some(contested);
7237 }
7238
7239 async fn stop_unresolved_review(
7240 &mut self,
7241 why: &str,
7242 shell: &[String],
7243 worktree: &Path,
7244 ) -> Result<()> {
7245 self.stop_reviewing_inner(why, shell, worktree, true).await
7246 }
7247
7248 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7249 self.stop_reviewing_inner(why, shell, worktree, false).await
7250 }
7251
7252 async fn stop_reviewing_inner(
7253 &mut self,
7254 why: &str,
7255 shell: &[String],
7256 worktree: &Path,
7257 repairable: bool,
7258 ) -> Result<()> {
7259 let round_idx = self.state.reviews.len() - 1;
7260 let needs_catchup_run = matches!(
7268 self.state.reviews[round_idx].e2e_status(),
7269 E2eStatus::Deferred | E2eStatus::ResourceBlocked
7270 );
7271 if needs_catchup_run {
7272 let round = self.state.reviews[round_idx].round;
7273 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7274 let commands = self.state.config.verify.e2e.clone();
7275 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7276 let cache_dir = self.state.config.cache_dir();
7277 let context = format!(
7278 "round {round}: verification unresolved, catching up before the final decision"
7279 );
7280 let wait = self.repository_wait();
7281 let (outcomes, verify_retried) = with_cache_lease(
7282 &mut self.state,
7283 cache_dir.as_deref(),
7284 "e2e",
7285 "e2e",
7286 worktree,
7287 &attempted_head,
7288 timeout,
7289 &context,
7290 &wait,
7291 |state, budget| {
7292 let shell = shell.to_vec();
7293 let commands = commands.clone();
7294 let context = context.clone();
7295 async move {
7296 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7297 .await
7298 }
7299 },
7300 )
7301 .await?;
7302 let last = &mut self.state.reviews[round_idx];
7303 last.e2e = outcomes;
7304 last.verify_retried = verify_retried;
7305 last.verified_head = Some(attempted_head);
7312 last.verified_at = Some(Timestamp::now());
7313 if verify_inconclusive(&last.e2e) {
7314 self.state.save()?;
7321 return Ok(());
7322 }
7323 last.e2e_deferred = false;
7324 }
7325 let last = &self.state.reviews[round_idx];
7326 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7327
7328 match last.e2e_status() {
7329 E2eStatus::Failed => {
7330 let red: Vec<String> = last
7331 .e2e
7332 .iter()
7333 .filter(|o| !o.ok())
7334 .map(|o| {
7335 format!(
7336 "`{}` -> {:?}\n{}",
7337 o.command,
7338 o.code,
7339 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7340 )
7341 })
7342 .collect();
7343 let can_repair = last.blocking == 0 && last.fix.is_none();
7344 self.state.reviews[round_idx].clean = false;
7345 self.state
7346 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7347 self.state.status = RunStatus::Blocked;
7348 if repairable
7349 && can_repair
7350 && self.state.reviews.len() < self.state.config.graph.review_rounds
7351 {
7352 self.state.status = RunStatus::Reviewing;
7353 }
7354 }
7355 E2eStatus::ResourceBlocked => {
7360 self.state.event(
7361 "review",
7362 format!(
7363 "{why}; e2e could not run (resource unavailable); not \
7364 deciding yet"
7365 ),
7366 );
7367 }
7368 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7369 self.state.event(
7370 "review",
7371 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7372 );
7373 self.record_contested_handoff();
7374 self.state.status = RunStatus::Gating;
7375 }
7376 }
7377 self.state.save()?;
7378 Ok(())
7379 }
7380
7381 async fn gate(&mut self) -> Result<()> {
7384 if self.hold_base_sync().await? {
7385 return Ok(());
7386 }
7387 if self.state.status == RunStatus::Failed
7399 || self
7400 .state
7401 .base_sync
7402 .as_ref()
7403 .is_some_and(|s| s.conflict.is_some())
7404 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7405 != Some(RunStatus::Gating)
7406 {
7407 return Ok(());
7408 }
7409 let Some(winner) = self.state.winner().cloned() else {
7410 return Ok(());
7411 };
7412 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7413 self.state.reconcile_verification_head(&head);
7414 self.state.save()?;
7415 let wait = self.repository_wait();
7416 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7417 return Ok(());
7418 }
7419 if self.state.gate_ran {
7420 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7431 self.state.status = RunStatus::Blocked;
7432 self.state.save()?;
7433 }
7434 return Ok(());
7435 }
7436 self.state.status = RunStatus::Gating;
7437 let mut outcomes = self.run_gate(&winner).await?;
7438 let mut attempted_head = self.state.gate_head.clone();
7439 loop {
7440 if verify_inconclusive(&outcomes) {
7451 self.state.save()?;
7452 return Ok(());
7453 }
7454 if outcomes.iter().all(CommandOutcome::ok) {
7455 break;
7456 }
7457 let fixes_before = self.state.gate_fixes.len();
7458 match self.gate_fix_round(&winner, &outcomes).await? {
7459 GateFix::Retry => {
7460 outcomes = self.run_gate(&winner).await?;
7461 attempted_head = self.state.gate_head.clone();
7462 }
7463 GateFix::Stop => {
7464 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
7465 if self.state.gate_fixes.len() > fixes_before
7466 && attempted_head.as_deref() != Some(current.as_str())
7467 && verify_current_head(&mut self.state, &winner, &wait).await?
7468 {
7469 outcomes = self.run_gate(&winner).await?;
7472 attempted_head = self.state.gate_head.clone();
7473 } else {
7474 break;
7475 }
7476 }
7477 GateFix::Defer => {
7478 self.state.save()?;
7479 return Ok(());
7480 }
7481 }
7482 }
7483 let passed = outcomes.iter().all(CommandOutcome::ok);
7484 self.state.gate = outcomes;
7485 self.state.gate_ran = true;
7486 self.state.gate_head = attempted_head;
7487 self.state.save()?;
7488 if !verify_current_head(&mut self.state, &winner, &wait).await? {
7489 return Ok(());
7490 }
7491 if !passed {
7492 self.state.status = RunStatus::Blocked;
7493 let spent = self.state.gate_fixes.len();
7494 self.state.event(
7495 "gate",
7496 if spent == 0 {
7497 "gate failed; not merging".to_owned()
7498 } else {
7499 format!("gate failed after {spent} gate-fix round(s); not merging")
7500 },
7501 );
7502 }
7503 self.state.save()?;
7504 Ok(())
7505 }
7506
7507 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7517 let commands = state.config.verify.pre_gate.clone();
7518 if commands.is_empty() {
7519 return;
7520 }
7521 let shell = state.config.shell();
7522 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7523 let (outcomes, _) = run_commands(
7524 state,
7525 "pre_gate",
7526 "pre_gate",
7527 0,
7528 &shell,
7529 &commands,
7530 &winner.worktree,
7531 timeout,
7532 )
7533 .await;
7534 for o in &outcomes {
7535 if !o.ok() {
7536 tracing::warn!(
7537 "pre_gate `{}` failed ({:?}); the gate decides",
7538 o.command,
7539 o.code
7540 );
7541 }
7542 state.event(
7543 "pre_gate",
7544 format!(
7545 "`{}` -> {}",
7546 o.command,
7547 if o.ok() {
7548 "pass".to_owned()
7549 } else {
7550 format!(
7551 "FAIL ({:?})\n{}",
7552 o.code,
7553 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
7554 )
7555 }
7556 ),
7557 );
7558 }
7559 state.pre_gate = outcomes;
7560 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7561 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7562 Ok(head) => {
7563 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7564 state.pre_gate_commit = Some(head);
7565 }
7566 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7567 },
7568 Ok(false) => {}
7569 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7570 }
7571 if let Err(e) = state.save() {
7572 tracing::warn!("could not persist the pre_gate record: {e:#}");
7573 }
7574 }
7575
7576 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7579 let wait = self.repository_wait();
7580 run_landing_gate(&mut self.state, winner, &wait).await
7581 }
7582
7583 async fn gate_fix_round(
7596 &mut self,
7597 winner: &Candidate,
7598 outcomes: &[CommandOutcome],
7599 ) -> Result<GateFix> {
7600 let cap = self.state.config.graph.gate_fix_rounds;
7601 let spent = self.state.gate_fixes.len();
7602 if spent >= cap {
7603 if cap > 0 {
7604 self.state.event(
7605 "gate",
7606 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7607 );
7608 }
7609 return Ok(GateFix::Stop);
7610 }
7611 if !gate_fixable(outcomes) {
7612 self.state.event(
7613 "gate",
7614 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7615 command or similar); not spending a fix round on it",
7616 );
7617 return Ok(GateFix::Stop);
7618 }
7619 let min_free = self.state.config.disk.min_free_bytes;
7620 if min_free > 0 {
7621 match crate::disk::free_bytes(&winner.worktree) {
7622 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7623 Ok(free) => {
7624 self.state.event(
7625 "gate",
7626 format!(
7627 "only {free} bytes free ({min_free} required by `[disk] \
7628 min_free_bytes`); not spending a fix round on a failure the disk \
7629 may explain"
7630 ),
7631 );
7632 return Ok(GateFix::Stop);
7633 }
7634 Err(e) => {
7635 self.state.event(
7636 "gate",
7637 format!("free disk space could not be measured ({e:#}); no fix round"),
7638 );
7639 return Ok(GateFix::Stop);
7640 }
7641 }
7642 }
7643
7644 let attempt = spent + 1;
7645 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7646 let base = self.landing_base();
7647 let fix_prompt = prompt::gate_fix_in(
7648 &self.state.instruction,
7649 &failed,
7650 attempt,
7651 cap,
7652 &self.state.config.graph.language,
7653 &self.state.config.graph.github_language,
7654 );
7655 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7656 let sessions = self.state.config.graph.sessions;
7657 let artifacts = self.merge_repair_artifacts();
7658 self.state.event(
7659 "gate",
7660 format!("gate failed; gate-fix round {attempt} of {cap}"),
7661 );
7662 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7663 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7664 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7667 head: before.clone(),
7668 passed: false,
7669 outcomes: Vec::new(),
7670 });
7671 self.state.save()?;
7672 let (_, seat, out) = self
7673 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7674 prompt: fix_prompt.clone(),
7675 spec,
7676 seat,
7677 cwd: winner.worktree.clone(),
7678 timeout,
7679 allow_write: true,
7680 sessions,
7681 artifacts: artifacts.clone(),
7682 stem: format!("gate-fix-{attempt}"),
7683 handover: None,
7684 })
7685 .await;
7686 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7689 return Ok(GateFix::Defer);
7690 }
7691 let mut record = GateFixRecord {
7692 agent: seat.agent.clone(),
7693 failed,
7694 notes: String::new(),
7695 committed: false,
7696 error: None,
7697 };
7698 match out {
7699 AgentOutcome::Ok(o) => {
7700 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7703 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7704 }
7705 }
7706 AgentOutcome::Dropped(_) => {
7707 record.error = Some("the CLI dropped the stream".to_owned());
7708 }
7709 AgentOutcome::Quota(o) => {
7710 self.state.quota.push(QuotaLoss {
7711 seat: seat.key.clone(),
7712 node: "gate-fix".to_owned(),
7713 at: Timestamp::now(),
7714 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7715 });
7716 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7717 }
7718 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7719 }
7720 self.state.seats.insert(seat.key.clone(), seat);
7721 let coordination = self.coordinate(&winner.worktree).await?;
7722 if let Ok(r) = git::rescue_commit(
7723 &winner.worktree,
7724 &format!("magi: gate fix {attempt} (uncommitted work)"),
7725 )
7726 .await
7727 {
7728 self.state.note_withheld("gate-fix", &r.withheld);
7729 }
7730 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7731 record.committed = after != before;
7732 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7733 let note = record.error.clone();
7734 self.state.gate_fixes.push(record);
7735 self.state.save()?;
7736 if !changed {
7737 self.state.event(
7738 "gate",
7739 match note {
7740 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7741 None => format!("gate-fix round {attempt}: the tree did not change"),
7742 },
7743 );
7744 return Ok(GateFix::Stop);
7745 }
7746 self.state.event(
7747 "gate",
7748 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7749 );
7750
7751 drop(coordination);
7752 let wait = self.repository_wait();
7753 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7754 }
7755
7756 async fn guarded_pr_message(
7763 &mut self,
7764 winner: &Candidate,
7765 facts: Option<&BranchFacts>,
7766 posting: bool,
7767 ) -> Result<Option<PrMessage>> {
7768 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7769 let head = match final_head.as_deref() {
7770 Some(h) => crate::pr_outcome::FinalHead::At(h),
7771 None => crate::pr_outcome::FinalHead::Unknown,
7772 };
7773 let summarized = if posting {
7775 Some(self.summarize_change(winner, final_head.as_deref()).await)
7776 } else {
7777 None
7778 };
7779 if self.state.resource_block.is_some() {
7784 self.state.event(
7785 "pr-summary",
7786 "a shared resource was busy; nothing was published, resume to summarize again",
7787 );
7788 return Ok(None);
7789 }
7790 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7791 anyhow::bail!(
7792 "the final commit moved during public description generation; retry before publishing"
7793 );
7794 }
7795 let note = match (&summarized, &self.state.change_summary) {
7796 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7797 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7798 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7799 (None, _) => ChangeNote::NotAsked,
7800 };
7801 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7802 if !posting {
7803 let identity = crate::scrub::Identity::current();
7804 return Ok(Some(PrMessage {
7805 title: crate::scrub::scrub(&pr.title, &identity),
7806 body: crate::scrub::scrub(&pr.body, &identity),
7807 }));
7808 }
7809 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7812 if let Some(known) = self
7813 .state
7814 .github_text
7815 .clone()
7816 .filter(|g| g.fingerprint == raw_fp)
7817 {
7818 return self.settle_withheld(winner, &pr, known).await;
7819 }
7820 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7821 let mut violations =
7822 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7823 let identity = crate::scrub::Identity::current();
7824 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7825 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7826 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7827 {
7828 violations.push(crate::github_text::Violation::SensitiveData);
7829 }
7830 let has_language = violations
7834 .iter()
7835 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7836 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7837 self.state.event(
7838 "github-text",
7839 if has_language {
7840 format!("description rejected: {violations:?}; requesting one rewrite")
7841 } else {
7842 format!("description rejected: {violations:?}; asking the owner")
7843 },
7844 );
7845 let mut rewritten = false;
7846 let mut rejected_rewrite = None;
7847 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7848 {
7849 let output = text;
7850 if let Some(title) = summary_title(&output) {
7851 let raw_body = summary_without_title(&output);
7854 let clean_title = crate::scrub::scrub(&title, &identity);
7855 let mut body = crate::scrub::scrub(&raw_body, &identity);
7856 let redacted = clean_title != title || body != raw_body;
7857 let title = clean_title;
7858 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7859 if !complete {
7860 self.state.event(
7861 "github-text",
7862 "description rewrite omitted required findings",
7863 );
7864 }
7865 if !body.trim().is_empty() {
7866 body.push_str("\n\n");
7867 body.push_str(&public_pr_evidence(&self.state, head));
7868 }
7869 let marker = format!("magi:run/{}", self.state.id);
7870 if !body.trim().is_empty() && !body.contains(&marker) {
7871 body.push_str(&format!("\n\n{marker}"));
7872 }
7873 let rewrite = PrMessage {
7874 title: title.clone(),
7875 body: body.clone(),
7876 };
7877 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7878 let mut failed = crate::github_text::check_for(
7879 &self.state.config.graph,
7880 &title,
7881 &body,
7882 redecision,
7883 );
7884 if (!crate::github_text::shareable(&title)
7885 || !crate::github_text::shareable(&body))
7886 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7887 {
7888 failed.push(crate::github_text::Violation::SensitiveData);
7889 }
7890 if complete
7891 && !title.trim().is_empty()
7892 && !body.trim().is_empty()
7893 && !failed.is_empty()
7894 {
7895 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7896 title: title.clone(),
7897 body: body.clone(),
7898 failed: failed
7899 .iter()
7900 .map(|v| crate::github_text::category(*v).to_owned())
7901 .collect(),
7902 });
7903 }
7904 if complete
7905 && !title.trim().is_empty()
7906 && !raw_body.trim().is_empty()
7907 && failed.is_empty()
7908 {
7909 decision = redecision;
7910 pr = PrMessage { title, body };
7911 rewritten = true;
7912 self.state
7913 .event("github-text", "description rewrite passed");
7914 if redacted {
7915 self.state.event(
7916 "github-text",
7917 "sensitive data removed from description rewrite",
7918 );
7919 }
7920 }
7921 }
7922 }
7923 if !rewritten {
7924 self.state.event(
7925 "github-text",
7926 "description rewrite unavailable or rejected; asking the owner",
7927 );
7928 if let Some(w) =
7929 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7930 {
7931 self.retire_github_text_question("the withheld text changed");
7934 let record = crate::run::GithubTextAsk {
7935 rewrite: rejected_rewrite,
7936 fingerprint: raw_fp,
7937 title: w.title,
7938 body: w.body,
7939 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7940 question: None,
7941 asks: 0,
7942 resolved: false,
7943 chosen_title: None,
7944 choice: None,
7945 };
7946 return self.settle_withheld(winner, &pr, record).await;
7947 }
7948 }
7949 }
7950 let (title, body) =
7951 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7952 Ok(Some(PrMessage { title, body }))
7953 }
7954
7955 async fn summarize_change(
7966 &mut self,
7967 winner: &Candidate,
7968 head: Option<&str>,
7969 ) -> std::result::Result<(), String> {
7970 let Some(head) = head else {
7971 return Err("the final commit could not be read".to_owned());
7972 };
7973 let remote = self.state.config.merge.remote.clone();
7974 let tracking = format!("{remote}/{}", self.state.base_branch);
7975 if !git::rev_exists(&winner.worktree, &tracking).await {
7976 return Err(format!("the remote base `{tracking}` could not be read"));
7977 }
7978 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7979 Ok(b) if !b.is_empty() => b,
7980 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7981 };
7982 let evidence = crate::pr_outcome::summary_evidence(
7983 &self.state,
7984 crate::pr_outcome::FinalHead::At(head),
7985 );
7986 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7987 if self
7988 .state
7989 .change_summary
7990 .as_ref()
7991 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7992 {
7993 return Ok(());
7994 }
7995 if self.state.change_summary.take().is_some() {
7996 self.state.event(
7997 "pr-summary",
7998 "the final diff or evidence changed; the earlier change summary was discarded",
7999 );
8000 }
8001 if let Err(e) = write_artifact(
8003 &self.state,
8004 &format!("public-description-{fingerprint}.md"),
8005 &format!("Final diff: {base}..{head}\n\n{evidence}"),
8006 ) {
8007 tracing::warn!("could not save public description evidence: {e:#}");
8008 }
8009 let chain = match agent::pick_chain(
8010 &self.state.config.agents,
8011 self.state.config.roles.synthesizer.as_ref(),
8012 &agent::installed,
8013 "synthesizer",
8014 ) {
8015 Ok(chain) => chain,
8016 Err(e) => {
8017 tracing::warn!("no agent for the change summary: {e:#}");
8018 self.state.event("pr-summary", "no summary agent available");
8019 return Err("no agent was available to summarize it".to_owned());
8020 }
8021 };
8022 let subjects = git::subjects(&winner.worktree, &base, head)
8023 .await
8024 .unwrap_or_default();
8025 let stat = git::diff_stat(&winner.worktree, &base, head)
8026 .await
8027 .unwrap_or_default();
8028 let stat: String = stat.chars().take(6_000).collect();
8029 let mut prompt = crate::prompt::change_summary_in(
8030 &base,
8031 head,
8032 &subjects,
8033 stat.trim(),
8034 &self.state.config.graph.github_language,
8035 );
8036 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8037 let sessions = self.state.config.graph.sessions;
8038 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8039 let prompts = self.state.config.prompts.clone();
8040 let cache = self.state.config.cache_dir();
8041 let run = self.state.id.clone();
8042 for (i, spec) in chain.into_iter().enumerate() {
8043 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8044 let github_spec = match agent::for_github_text(
8045 &spec,
8046 &self.state.config.graph.github_language,
8047 &winner.worktree,
8048 ) {
8049 Ok(spec) => spec,
8050 Err(e) => {
8051 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8052 continue;
8053 }
8054 };
8055 let job = SeatJob {
8056 spec: github_spec,
8057 seat,
8058 cwd: winner.worktree.clone(),
8059 prompt: prompt.clone(),
8060 timeout,
8061 allow_write: false,
8062 sessions,
8063 artifacts: agent::artifacts_dir(&self.state.dir()),
8064 stem: if i == 0 {
8065 "change-summary".to_owned()
8066 } else {
8067 format!("change-summary-{}", spec.id)
8068 },
8069 handover: None,
8070 };
8071 let ctx = WaveCtx {
8072 run: &run,
8073 node: "pr-summary",
8074 prompts: &prompts,
8075 cache: cache.as_deref(),
8076 round: None,
8077 carry_seats: false,
8078 pauses: Some((&self.pause, &self.interrupt)),
8079 };
8080 let (seat, outcome) =
8081 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8082 self.state.seats.insert(seat.key.clone(), seat);
8083 let text = match outcome {
8084 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8085 AgentOutcome::ResourceBlocked(why) => {
8086 if self.state.resource_block.is_none() {
8089 self.state.resource_block = Some(crate::run::ResourceBlock {
8090 cause: crate::run::ResourceBlockCause::Other,
8091 node: "pr-summary".to_owned(),
8092 kind: "shared resource".to_owned(),
8093 detail: why,
8094 at: Timestamp::now(),
8095 });
8096 }
8097 return Err("a shared resource was busy".to_owned());
8098 }
8099 _ => continue,
8100 };
8101 let blind = &self.state.config.blind;
8102 let Some(title) = summary_title(&text) else {
8103 continue;
8104 };
8105 let body: String = summary_without_title(&text)
8106 .lines()
8107 .filter(|l| !l.contains("magi:run/"))
8108 .collect::<Vec<_>>()
8109 .join("\n");
8110 let body = blind::sanitize_prose(body.trim(), blind);
8111 let title = blind::sanitize_prose(&title, blind);
8112 if body.trim().is_empty() || title.trim().is_empty() {
8113 continue;
8114 }
8115 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8116 continue;
8117 }
8118 if git::rev_parse(&winner.worktree, "HEAD")
8120 .await
8121 .ok()
8122 .as_deref()
8123 != Some(head)
8124 {
8125 return Err("the branch moved while the summary was written".to_owned());
8126 }
8127 if git::merge_base(&winner.worktree, &tracking, head)
8128 .await
8129 .ok()
8130 .as_deref()
8131 != Some(base.as_str())
8132 || crate::pr_outcome::summary_evidence(
8133 &self.state,
8134 crate::pr_outcome::FinalHead::At(head),
8135 ) != evidence
8136 {
8137 return Err("the summary evidence changed while it was written".to_owned());
8138 }
8139 self.state.change_summary = Some(crate::run::ChangeSummary {
8140 head: head.to_owned(),
8141 base,
8142 evidence: fingerprint,
8143 title,
8144 body,
8145 });
8146 self.state.event(
8147 "pr-summary",
8148 format!("change summary written by {} for the final diff", spec.id),
8149 );
8150 if let Err(e) = self.state.save() {
8151 tracing::warn!("could not save the change summary: {e:#}");
8152 }
8153 return Ok(());
8154 }
8155 self.state
8156 .event("pr-summary", "no agent produced a change summary");
8157 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8158 }
8159
8160 async fn rewrite_pr_text(
8170 &mut self,
8171 winner: &Candidate,
8172 pr: &PrMessage,
8173 violations: &[crate::github_text::Violation],
8174 ) -> Option<String> {
8175 let prompt = format!(
8176 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8177 pr.title, pr.body
8178 );
8179 let sessions = self.state.config.graph.sessions;
8180 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8181 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8184 let key = format!("impl-{}", winner.label);
8185 if let Some(spec) = self
8186 .state
8187 .config
8188 .agents
8189 .iter()
8190 .find(|a| a.id == winner.agent)
8191 .cloned()
8192 && let Some(existing) = self.state.seats.get(&key)
8193 && existing.agent == spec.id
8194 && agent::has_session(spec.kind, existing, sessions)
8195 {
8196 attempts.push((spec, Some(existing.clone())));
8197 }
8198 match agent::pick_chain(
8199 &self.state.config.agents,
8200 self.state.config.roles.language_judge.as_ref(),
8201 &agent::installed,
8202 "language_judge",
8203 ) {
8204 Ok(chain) => {
8205 for spec in chain {
8206 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8207 continue;
8208 }
8209 attempts.push((spec, None));
8210 }
8211 }
8212 Err(e) if attempts.is_empty() => {
8213 tracing::warn!("no agent for the description rewrite: {e:#}");
8214 self.state
8215 .event("github-text", "no rewrite agent available");
8216 return None;
8217 }
8218 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8219 }
8220 let prompts = self.state.config.prompts.clone();
8221 let cache = self.state.config.cache_dir();
8222 let run = self.state.id.clone();
8223 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8224 let resumed = resumed_seat.is_some();
8225 let seat = resumed_seat.unwrap_or_else(|| {
8226 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8227 });
8228 let github_spec = match agent::for_github_text(
8229 &spec,
8230 &self.state.config.graph.github_language,
8231 &winner.worktree,
8232 ) {
8233 Ok(spec) => spec,
8234 Err(e) => {
8235 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8236 continue;
8237 }
8238 };
8239 let job = SeatJob {
8240 spec: github_spec,
8241 seat,
8242 cwd: winner.worktree.clone(),
8243 prompt: prompt.clone(),
8244 timeout: retry_budget(timeout_full, resumed),
8245 allow_write: false,
8246 sessions,
8247 artifacts: agent::artifacts_dir(&self.state.dir()),
8248 stem: if i == 0 {
8249 "github-text-rewrite".to_owned()
8250 } else {
8251 format!("github-text-rewrite-{}", spec.id)
8252 },
8253 handover: None,
8254 };
8255 let ctx = WaveCtx {
8256 run: &run,
8257 node: "github-text",
8258 prompts: &prompts,
8259 cache: cache.as_deref(),
8260 round: None,
8261 carry_seats: false,
8262 pauses: Some((&self.pause, &self.interrupt)),
8263 };
8264 let (seat, outcome) =
8265 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8266 self.state.seats.insert(seat.key.clone(), seat);
8267 match outcome {
8268 AgentOutcome::Ok(o)
8269 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8270 {
8271 return Some(o.text);
8272 }
8273 AgentOutcome::ResourceBlocked(_) => return None,
8274 _ => {}
8275 }
8276 }
8277 None
8278 }
8279
8280 fn retire_github_text_question(&mut self, why: &str) {
8282 let Some(id) = self
8283 .state
8284 .github_text
8285 .as_ref()
8286 .and_then(|g| g.question.clone())
8287 else {
8288 return;
8289 };
8290 let _ = ask::Questions::open().update(&id, |q| {
8291 q.abandon(why);
8292 Ok(())
8293 });
8294 }
8295
8296 async fn settle_withheld(
8300 &mut self,
8301 winner: &Candidate,
8302 pr: &PrMessage,
8303 mut g: crate::run::GithubTextAsk,
8304 ) -> Result<Option<PrMessage>> {
8305 use crate::github_text as gt;
8306 let store = ask::Questions::open();
8307 let timeout = self.state.config.graph.answer_timeout;
8308 if !g.resolved {
8309 let mut standing = g
8310 .question
8311 .as_ref()
8312 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8313 if let Some(q) = standing.as_ref()
8314 && q.status.open()
8315 && gt::expired(q, timeout)
8316 {
8317 let why = format!("no answer within {}s of asking", timeout.max(1));
8318 standing = store
8319 .update(&q.id, |q| {
8320 q.abandon(&why);
8321 Ok(())
8322 })
8323 .ok()
8324 .map(|(q, ())| q);
8325 }
8326 let mut ask_again: Option<Option<Vec<String>>> = None;
8327 match standing {
8328 None if g.asks <= 1 => {
8331 g.asks = 0;
8332 ask_again = Some(None);
8333 }
8334 None => {
8335 g.resolved = true;
8336 g.chosen_title = None;
8337 g.choice = Some(GithubTextChoice::Fallback);
8338 }
8339 Some(q) if q.status.open() => {
8340 self.park_on_github_text(&g)?;
8341 return Ok(None);
8342 }
8343 Some(q) => match gt::read_reply(&q) {
8344 gt::Reply::Fallback => {
8345 g.resolved = true;
8346 g.chosen_title = None;
8347 g.choice = Some(GithubTextChoice::Fallback);
8348 }
8349 gt::Reply::Original => {
8350 g.resolved = true;
8351 g.chosen_title = None;
8352 g.choice = Some(GithubTextChoice::Original);
8353 }
8354 gt::Reply::Rewrite => {
8355 g.resolved = true;
8356 g.chosen_title = None;
8357 g.choice = Some(if g.rewrite.is_some() {
8358 GithubTextChoice::Rewrite
8359 } else {
8360 GithubTextChoice::Fallback
8361 });
8362 }
8363 gt::Reply::Title(text) => {
8364 let first = text
8365 .lines()
8366 .map(str::trim)
8367 .find(|l| !l.is_empty())
8368 .unwrap_or("");
8369 let decision = if first.is_empty() {
8370 None
8371 } else {
8372 let candidate = PrMessage {
8373 title: first.to_owned(),
8374 body: String::new(),
8375 };
8376 self.judge_pr_language(&winner.worktree, &candidate).await
8377 };
8378 match gt::vet_title(
8379 &text,
8380 decision,
8381 self.state.config.graph.github_english(),
8382 ) {
8383 Ok(title) => {
8384 g.resolved = true;
8385 g.chosen_title = Some(title);
8386 g.choice = Some(GithubTextChoice::MyTitle);
8387 }
8388 Err(cats) if g.asks < 2 => {
8389 ask_again = Some(Some(
8390 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8391 ))
8392 }
8393 Err(_) => {
8394 g.resolved = true;
8395 g.chosen_title = None;
8396 g.choice = Some(GithubTextChoice::Fallback);
8397 }
8398 }
8399 }
8400 },
8401 }
8402 if let Some(rejected) = ask_again {
8403 let w = gt::Withheld {
8406 title: g.title,
8407 body: g.body,
8408 categories: g.categories.clone(),
8409 };
8410 let artifact = match crate::run::write_artifact(
8413 &self.state,
8414 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8415 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8416 ) {
8417 Ok(p) => Some(p),
8418 Err(e) => {
8419 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8420 None
8421 }
8422 };
8423 let (detail, comparison) = gt::question_content(
8424 &w,
8425 &pr.title,
8426 &pr.body,
8427 &self.state.id,
8428 rejected.as_deref(),
8429 artifact.as_deref(),
8430 g.rewrite.as_ref(),
8431 );
8432 let mut q = ask::Question::new(
8433 self.state.id.clone(),
8434 gt::ASK_NODE.to_owned(),
8435 gt::ASK_SEAT.to_owned(),
8436 gt::question_summary(&self.state.config.graph.language, &w),
8437 detail,
8438 gt::question_choices(&w, g.rewrite.as_ref()),
8439 );
8440 q.posting_comparison = Some(comparison);
8441 q.answer_timeout = timeout;
8442 g.question = Some(q.id.clone());
8445 g.asks += 1;
8446 self.state.github_text = Some(g.clone());
8447 self.state.event(
8448 "github-text",
8449 format!("asking the owner about the withheld text ({})", q.short()),
8450 );
8451 self.state.save()?;
8452 store
8453 .put_executable(&mut q)
8454 .context("file the posting-gate question")?;
8455 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8456 tracing::warn!(
8457 "could not notify about posting-gate question {}: {e:#}",
8458 q.short()
8459 );
8460 }
8461 self.park_on_github_text(&g)?;
8462 return Ok(None);
8463 }
8464 }
8465 let resolved = g.chosen_title.clone();
8466 let original = g.choice == Some(GithubTextChoice::Original);
8467 self.state.event(
8469 "github-text",
8470 if original {
8471 format!(
8472 "posting the original text for: {}",
8473 match (g.title, g.body) {
8474 (true, true) => "title, description",
8475 (true, false) => "title",
8476 _ => "description",
8477 }
8478 )
8479 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8480 "posting the saved rewrite title and description".to_owned()
8481 } else if resolved.is_some() {
8482 "posting the owner's replacement title".to_owned()
8483 } else {
8484 "posting the neutral text for the withheld fields".to_owned()
8485 },
8486 );
8487 self.state.github_text = Some(g.clone());
8488 self.state.save()?;
8489 if g.choice == Some(GithubTextChoice::Rewrite)
8490 && let Some(r) = g.rewrite.as_ref()
8491 {
8492 return Ok(Some(PrMessage {
8493 title: r.title.clone(),
8494 body: r.body.clone(),
8495 }));
8496 }
8497 let w = gt::Withheld {
8498 title: g.title,
8499 body: g.body,
8500 categories: g.categories.clone(),
8501 };
8502 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8503 if let Some(t) = resolved {
8504 title = t;
8505 }
8506 let english = gt::LanguageDecision {
8508 title_english: true,
8509 body_english: true,
8510 body_complete: true,
8511 };
8512 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8513 if original {
8514 if g.title {
8517 title = pr.title.clone();
8518 }
8519 if g.body {
8520 body = pr.body.clone();
8521 }
8522 }
8523 Ok(Some(PrMessage { title, body }))
8524 }
8525
8526 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8527 self.state.github_text = Some(g.clone());
8528 self.state.end_pass_parked(ParkCause::Question);
8529 self.state.parked = true;
8530 self.state.event(
8531 "github-text",
8532 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8533 );
8534 self.state.save()
8535 }
8536
8537 async fn judge_pr_language(
8540 &mut self,
8541 cwd: &Path,
8542 pr: &PrMessage,
8543 ) -> Option<crate::github_text::LanguageDecision> {
8544 self.state.config.roles.language_judge.as_ref()?;
8545 if !self.state.config.graph.language_gate() {
8546 return None;
8547 }
8548 let decision =
8549 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8550 self.state.event(
8551 "github-text",
8552 if decision.is_some() {
8553 "language decided by the language judge"
8554 } else {
8555 "language judge unavailable; using built-in heuristics"
8556 },
8557 );
8558 decision
8559 }
8560
8561 async fn merge(&mut self) -> Result<()> {
8562 if self.resource_halt()? || self.hold_base_sync().await? {
8563 return Ok(());
8564 }
8565 if self
8580 .state
8581 .base_sync
8582 .as_ref()
8583 .is_some_and(|s| s.conflict.is_some())
8584 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8585 != Some(RunStatus::Gating)
8586 || !self.state.gate_allows_progress()
8594 {
8595 return Ok(());
8596 }
8597 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8606 return Ok(());
8607 }
8608 let Some(winner) = self.state.winner().cloned() else {
8609 return Ok(());
8610 };
8611 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8612 if self.state.gate_head.as_deref() != Some(head.as_str()) {
8613 self.state.status = RunStatus::Blocked;
8614 self.state.event(
8615 "gate",
8616 "winner changed after gate; fresh verification required",
8617 );
8618 self.state.save()?;
8619 return Ok(());
8620 }
8621 let wait = self.repository_wait();
8622 if !verify_current_head(&mut self.state, &winner, &wait).await? {
8623 return Ok(());
8624 }
8625 let repo = self.state.repo.clone();
8626 let base = self.state.base_branch.clone();
8627 let mode = self.state.config.merge.mode;
8628 let style = self.state.config.merge.style;
8629 let facts = if is_review_run(&self.state) {
8630 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8631 let start = review_base(
8632 &repo,
8633 &self.state.config.merge.remote,
8634 &base,
8635 &self.state.base_commit,
8636 &winner.branch,
8637 )
8638 .await;
8639 branch_facts(&repo, &start, &winner.branch).await
8640 } else {
8641 None
8642 };
8643 let leaked = if is_review_run(&self.state) {
8646 leaked_subjects(&self.state, &winner.branch).await
8647 } else {
8648 Some(Vec::new())
8649 };
8650 let Some(pr) = self
8654 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8655 .await?
8656 else {
8657 return Ok(());
8658 };
8659 let message = pr.commit_message();
8660
8661 let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
8662 if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
8663 return Ok(());
8664 }
8665 let outcome = match mode {
8666 MergeMode::None => MergeOutcome {
8667 mode,
8668 ok: true,
8669 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8670 empty: false,
8671 },
8672 MergeMode::Pr | MergeMode::Local
8673 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8674 {
8675 MergeOutcome {
8676 mode,
8677 ok: false,
8678 detail: empty_candidate_detail(&self.state, &base),
8679 empty: true,
8680 }
8681 }
8682 MergeMode::Local => {
8683 let on = git::current_branch(&repo).await?;
8684 if on.as_deref() != Some(base.as_str()) {
8685 MergeOutcome {
8686 mode,
8687 ok: false,
8688 detail: format!(
8689 "{} has {} checked out, not the base branch {base}",
8690 repo.display(),
8691 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8692 ),
8693 empty: false,
8694 }
8695 } else if !git::is_clean(&repo).await? {
8696 MergeOutcome {
8697 mode,
8698 ok: false,
8699 detail: format!("{} is dirty; refusing to merge", repo.display()),
8700 empty: false,
8701 }
8702 } else {
8703 let out = match style {
8704 MergeStyle::Merge => {
8705 git::merge_no_ff(&repo, &winner.branch, &message).await?
8706 }
8707 MergeStyle::Squash => {
8708 git::merge_squash(&repo, &winner.branch, &message).await?
8709 }
8710 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8711 };
8712 MergeOutcome {
8713 mode,
8714 ok: out.ok(),
8715 detail: if out.ok() { out.stdout } else { out.stderr },
8716 empty: false,
8717 }
8718 }
8719 }
8720 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8721 self.state.pr_publication_pending = true;
8722 self.state.save()?;
8723 let out =
8724 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8725 self.state.pr_publication_pending = out.is_err();
8726 MergeOutcome {
8727 mode,
8728 ok: out.is_ok(),
8729 detail: match out {
8730 Ok(url) => url,
8731 Err(e) => format!("{e:#}"),
8732 },
8733 empty: false,
8734 }
8735 }
8736 MergeMode::Pr => {
8737 let remote = self.state.config.merge.remote.clone();
8738 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8739 let pushed = if self.state.foreign_checkout.is_some() {
8740 let refspec = format!("{delivery_head}:refs/heads/{public}");
8741 git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
8742 } else {
8743 git::push(&winner.worktree, &remote, &winner.branch).await?
8744 };
8745 if !pushed.ok() {
8746 MergeOutcome {
8747 mode,
8748 ok: false,
8749 detail: pushed.stderr,
8750 empty: false,
8751 }
8752 } else {
8753 if self.state.foreign_checkout.is_some() {
8754 self.state.pr_pushed_head = Some(delivery_head.clone());
8755 }
8756 let found = land::find_open_pr(&winner.worktree, &public, &base).await;
8762 let out = match pr_merge_plan(found) {
8763 PrPlan::Create => {
8764 gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
8765 .await
8766 }
8767 PrPlan::Adopt { url, title } => {
8768 self.state
8769 .event("merge", format!("Pr: adopted open pull request {url}"));
8770 if self.state.github_text.as_ref().is_some_and(|g| {
8771 g.resolved
8772 && g.choice == Some(GithubTextChoice::Rewrite)
8773 && g.rewrite.is_some()
8774 }) {
8775 land::set_pr_text_verbatim(
8778 &winner.worktree,
8779 &url,
8780 &pr.title,
8781 &pr.body,
8782 )
8783 .await
8784 .map(|()| url)
8785 } else {
8786 if title != pr.title
8787 && (!is_review_run(&self.state)
8788 || leaked
8789 .as_deref()
8790 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8791 && let Err(e) =
8792 if self.state.github_text.as_ref().is_some_and(|g| {
8793 g.resolved
8794 && g.title
8795 && g.choice == Some(GithubTextChoice::Original)
8796 }) {
8797 land::set_pr_title_verbatim(
8799 &winner.worktree,
8800 &url,
8801 &pr.title,
8802 )
8803 .await
8804 } else {
8805 land::set_pr_title(
8806 &mut self.state,
8807 &winner.worktree,
8808 &url,
8809 &pr.title,
8810 )
8811 .await
8812 }
8813 {
8814 tracing::warn!("could not refresh title of {url}: {e:#}");
8815 self.state
8816 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8817 }
8818 Ok(url)
8819 }
8820 }
8821 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8822 };
8823 match out {
8824 Ok(url) => MergeOutcome {
8825 mode,
8826 ok: true,
8827 detail: url,
8828 empty: false,
8829 },
8830 Err(e) => MergeOutcome {
8831 mode,
8832 ok: false,
8833 detail: e.to_string(),
8834 empty: false,
8835 },
8836 }
8837 }
8838 }
8839 };
8840
8841 self.state.status = match (mode, outcome.ok) {
8842 (MergeMode::None, _) => RunStatus::Ready,
8843 (_, true) if outcome.empty => RunStatus::Ready,
8844 (MergeMode::Pr, true) => RunStatus::Ready,
8845 (_, true) => RunStatus::Merged,
8846 (_, false) => RunStatus::Blocked,
8847 };
8848 self.state.event(
8849 "merge",
8850 format!(
8851 "{:?}: {}",
8852 mode,
8853 outcome.detail.lines().next().unwrap_or("")
8854 ),
8855 );
8856 self.state.merge = Some(outcome);
8857 self.state.save()?;
8858
8859 if self.state.config.graph.land
8865 && mode == MergeMode::Pr
8866 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8867 {
8868 self.run_land().await?;
8869 }
8870 self.settle_questions();
8875 Ok(())
8876 }
8877
8878 async fn run_land(&mut self) -> Result<()> {
8889 crate::pr_head::ensure_owner(&self.state)?;
8890 let url = self
8891 .state
8892 .pr
8893 .as_ref()
8894 .map(|pr| pr.url.clone())
8895 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8896 .unwrap_or_default();
8897 let url = url.lines().next().unwrap_or("").trim().to_owned();
8898 if !url.starts_with("http") {
8899 return Ok(());
8900 }
8901 loop {
8904 match land::land(&mut self.state, &url).await {
8905 Ok(_)
8906 if self
8907 .state
8908 .merge_repairs
8909 .last()
8910 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8911 {
8912 if self.repair_merge_review().await? {
8913 continue;
8914 }
8915 }
8916 Ok(pr) if self.state.parked => {
8917 let _ = pr;
8921 }
8922 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8923 self.state.status = RunStatus::Superseded;
8924 self.state.save()?;
8925 }
8926 Ok(pr) => {
8927 self.state.status = match pr.state {
8928 land::PrLifecycle::Merged => RunStatus::Merged,
8929 _ => RunStatus::Blocked,
8930 };
8931 if bump::should_release_bump(self.state.status)
8938 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8939 {
8940 self.state
8946 .event("bump", format!("release bump skipped: {e:#}"));
8947 }
8948 if self.state.status == RunStatus::Merged {
8951 crate::followup::after_merge(&mut self.state, &pr.url).await;
8952 }
8953 self.state.save()?;
8954 }
8955 Err(e) => {
8956 if e.is::<crate::repo_lock::ResourceBlocked>() {
8957 return Err(e);
8958 }
8959 self.state.status = RunStatus::Blocked;
8960 self.state.event("land", format!("gave up: {e}"));
8961 self.state.save()?;
8962 }
8963 }
8964 break;
8965 }
8966 Ok(())
8967 }
8968
8969 async fn repair_merge_review(&mut self) -> Result<bool> {
8972 use crate::run::MergeRepairPhase as Phase;
8973 crate::pr_head::ensure_owner(&self.state)?;
8974 let i = self.state.merge_repairs.len() - 1;
8975 let request = self.state.merge_repairs[i].clone();
8976 if request.phase == Phase::Fixing {
8977 let restored = match self.state.winner().cloned() {
8978 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8979 None => Err(anyhow::anyhow!("repair has no winning branch")),
8980 };
8981 let why = match restored {
8982 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8983 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8984 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8985 };
8986 self.hold_merge_repair(i, why)?;
8987 return Ok(false);
8988 }
8989 if request.phase == Phase::Reviewing && self.state.gate_ran {
8994 let winner = self
8995 .state
8996 .winner()
8997 .context("repair has no winning branch")?;
8998 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8999 if self.state.gate_head.as_deref() != Some(head.as_str())
9000 || !git::is_clean(&winner.worktree).await?
9001 {
9002 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9003 return Ok(false);
9004 }
9005 }
9006 let failure = if i >= self.state.config.graph.land_rounds
9007 || self.state.config.graph.review_rounds == 0
9008 {
9009 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9010 } else {
9011 None
9012 };
9013 if let Some(why) = failure {
9014 self.hold_merge_repair(i, why)?;
9015 return Ok(false);
9016 }
9017 if request.phase == Phase::Requested {
9018 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9019 Ok(pr) => pr,
9020 Err(e) => {
9021 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9022 return Ok(false);
9023 }
9024 };
9025 self.state.merge_repairs[i].phase = Phase::Fixing;
9026 self.state.land_approval = None;
9027 self.state.land_armed_head = None;
9028 self.state.save()?;
9029 let evidence = serde_json::to_string_pretty(&request)?;
9032 let reason = format!(
9033 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9034 );
9035 let budget = self.state.config.graph.land_rounds;
9036 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9037 match result {
9038 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9039 Ok(land::Fixed::Failed(why)) => {
9040 self.hold_merge_repair(
9041 i,
9042 format!(
9043 "repair failed: {why}; inspect the branch and run an explicit review"
9044 ),
9045 )?;
9046 return Ok(false);
9047 }
9048 Err(e) => {
9049 self.hold_merge_repair(
9050 i,
9051 format!(
9052 "repair failed: {e:#}; inspect the branch and run an explicit review"
9053 ),
9054 )?;
9055 return Ok(false);
9056 }
9057 }
9058 self.begin_merge_repair_review(i)?;
9062 }
9063 self.review_loop().await?;
9064 self.gate().await?;
9065 if self.resource_halt()? || self.state.parked {
9066 return Ok(false);
9067 }
9068 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9069 != Some(RunStatus::Gating)
9070 || !self.state.gate_allows_progress()
9071 {
9072 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9073 return Ok(false);
9074 }
9075 let winner = self
9078 .state
9079 .winner()
9080 .cloned()
9081 .context("repair has no winning branch")?;
9082 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9083 if self.state.gate_head.as_deref() != Some(after.as_str())
9084 || !git::is_clean(&winner.worktree).await?
9085 {
9086 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9087 return Ok(false);
9088 }
9089 let expected = self
9090 .state
9091 .pr_pushed_head
9092 .as_deref()
9093 .unwrap_or(&request.head)
9094 .to_owned();
9095 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9096 if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9099 return Ok(false);
9100 }
9101 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9102 let refspec = format!("{after}:refs/heads/{public}");
9103 let push = git::git_raw(
9104 &winner.worktree,
9105 &["push", &lease, &self.state.config.merge.remote, &refspec],
9106 )
9107 .await;
9108 match push {
9109 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9110 Ok(out) => {
9111 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9112 return Ok(false);
9113 }
9114 Err(e) => {
9115 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9116 return Ok(false);
9117 }
9118 }
9119 self.state.merge_repairs[i].phase = Phase::Complete;
9120 self.state.status = RunStatus::Landing;
9121 self.state.event(
9122 "repair",
9123 "new review and gate completed; fresh owner merge decision required",
9124 );
9125 self.state.save()?;
9126 Ok(true)
9127 }
9128
9129 fn merge_repair_artifacts(&self) -> PathBuf {
9130 let artifacts = agent::artifacts_dir(&self.state.dir());
9131 match self
9132 .state
9133 .merge_repairs
9134 .last()
9135 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9136 {
9137 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9138 None => artifacts,
9139 }
9140 }
9141
9142 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9143 if let Some(e2e) = self.state.head_e2e.take() {
9144 self.state
9145 .verification_history
9146 .push(crate::run::VerificationEvidence {
9147 superseded_by: None,
9148 archived_at: Some(Timestamp::now()),
9149 fixes_spent: Some(self.state.gate_fixes.len()),
9150 gate_head: None,
9151 gate_ran: false,
9152 gate: Vec::new(),
9153 e2e: Some(e2e),
9154 });
9155 }
9156 self.state.reviews.clear();
9157 self.state.gate.clear();
9158 self.state.gate_ran = false;
9159 self.state.gate_head = None;
9160 self.state.gate_fixes.clear();
9161 self.state.gate_repair_e2e = None;
9162 self.state.pre_gate.clear();
9163 self.state.pre_gate_commit = None;
9164 self.state.contested_handoff = None;
9165 self.state.land_reruns = None;
9166 self.state.land_approval = None;
9167 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9168 self.state.status = RunStatus::Reviewing;
9169 self.state.save()
9170 }
9171
9172 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9173 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9174 self.state.merge_repairs[index].failure = Some(why.clone());
9175 self.state.status = RunStatus::Blocked;
9176 self.state.event("repair", why);
9177 self.state.save()
9178 }
9179
9180 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9184 if let Some(existing) = self.state.seats.get(key)
9185 && existing.agent == agent
9186 {
9187 return existing.clone();
9188 }
9189 let fresh = if self.state.seats.contains_key(key) {
9192 handover_seat(key, agent, self.state.next_seat_seed())
9193 } else {
9194 SeatState::new(key, agent, self.state.seed)
9195 };
9196 self.state.seats.insert(key.to_owned(), fresh.clone());
9197 fresh
9198 }
9199
9200 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9206 match self.state.seats.get(key) {
9207 Some(s) if s.agent != spec.id => {
9208 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9209 }
9210 _ => spec,
9211 }
9212 }
9213
9214 fn view(&self, c: &Candidate) -> CandidateView {
9216 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9217 .unwrap_or_default();
9218 let (patch, _) = blind::sanitize_patch(
9219 &format!("candidate {} patch", c.label),
9220 &raw,
9221 &self.state.config.blind,
9222 );
9223 CandidateView {
9224 label: c.label,
9225 branch: c.branch.clone(),
9226 summary: c.summary.clone(),
9227 stat: c.stat.clone(),
9228 patch,
9229 }
9230 }
9231
9232 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9234 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9235 prompt::judge(
9236 "(see above)",
9237 &views,
9238 self.roles.judges.len(),
9239 base_short,
9240 "en",
9241 )
9242 }
9243
9244 fn vote_prompt_full(
9250 &self,
9251 j: usize,
9252 labels: &[char],
9253 language: &str,
9254 candidates: &[Candidate],
9255 base_short: &str,
9256 ) -> String {
9257 let mut text = format!(
9258 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9259 prompt::final_vote(labels, language),
9260 self.state.instruction,
9261 self.candidate_block(candidates, base_short)
9262 );
9263 if let Some(own) = self
9264 .state
9265 .judgements
9266 .get(j)
9267 .filter(|r| !r.ranking.is_empty())
9268 {
9269 let reasons = own
9270 .reasons
9271 .iter()
9272 .map(|(k, v)| format!("- {k}: {v}"))
9273 .collect::<Vec<_>>()
9274 .join("\n");
9275 text.push_str(&format!(
9276 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9277 own.ranking.iter().collect::<String>(),
9278 if reasons.is_empty() {
9279 ""
9280 } else {
9281 ", because:\n"
9282 }
9283 ));
9284 }
9285 if !self.state.deliberation.is_empty() {
9286 text.push_str("\n# What was argued before this vote\n");
9287 for t in self.transcript(&[], j) {
9288 text.push_str(&format!(
9289 "\n## {}{}\n\n{}\n",
9290 t.who,
9291 if t.is_self { " (you)" } else { "" },
9292 t.body.trim()
9293 ));
9294 }
9295 }
9296 text
9297 }
9298
9299 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9306 let mut turns = Vec::new();
9307 for j in &self.state.judgements {
9308 if j.ranking.is_empty() {
9309 continue;
9310 }
9311 let reasons = j
9312 .reasons
9313 .iter()
9314 .map(|(k, v)| format!("- {k}: {v}"))
9315 .collect::<Vec<_>>()
9316 .join("\n");
9317 turns.push(Turn {
9318 who: format!("Judge {} (opening ranking)", j.judge),
9319 is_self: j.judge == self_idx + 1,
9320 body: format!(
9321 "Ranked {}{}{reasons}",
9322 j.ranking.iter().collect::<String>(),
9323 if reasons.is_empty() {
9324 ""
9325 } else {
9326 ", because:\n"
9327 }
9328 ),
9329 });
9330 }
9331 for t in self
9332 .state
9333 .deliberation
9334 .iter()
9335 .flat_map(|r| r.turns.iter())
9336 .chain(current)
9337 {
9338 turns.push(Turn {
9339 who: format!("Judge {}", t.judge),
9340 is_self: t.judge == self_idx + 1,
9341 body: t.body.clone(),
9342 });
9343 }
9344 turns
9345 }
9346}
9347
9348fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9350 agent::has_session(spec.kind, seat, sessions)
9351}
9352
9353fn next_untried_in_roster<'a>(
9374 roster: &'a [AgentSpec],
9375 start: usize,
9376 tried: &BTreeSet<String>,
9377) -> Option<&'a AgentSpec> {
9378 roster
9379 .get(start + 1..)?
9380 .iter()
9381 .find(|s| !tried.contains(&s.id))
9382}
9383
9384fn pick_successor<'a>(
9405 roster: &'a [AgentSpec],
9406 start: usize,
9407 tried: &BTreeSet<String>,
9408 carried: Option<&BTreeSet<String>>,
9409 others: &BTreeSet<String>,
9410) -> Option<&'a AgentSpec> {
9411 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9412 roster
9413 .get(start + 1..)
9414 .and_then(|tail| {
9415 tail.iter()
9416 .filter(free)
9417 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9418 })
9419 .or_else(|| {
9420 carried
9421 .is_some()
9422 .then(|| roster.iter().find(free))
9423 .flatten()
9424 })
9425 .or_else(|| match carried {
9426 Some(c) => next_for_seat(roster, start, tried, c),
9427 None => next_untried_in_roster(roster, start, tried),
9428 })
9429}
9430
9431fn next_for_seat<'a>(
9441 roster: &'a [AgentSpec],
9442 start: usize,
9443 round_tried: &BTreeSet<String>,
9444 carried_failed: &BTreeSet<String>,
9445) -> Option<&'a AgentSpec> {
9446 roster
9447 .get(start + 1..)?
9448 .iter()
9449 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9450 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9451}
9452
9453fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9459 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9460 return spec;
9461 };
9462 let ok = |id: &str| !h.failed.contains(id);
9463 if let Some(last) = h.last_ok.as_deref()
9464 && ok(last)
9465 && let Some(s) = roster.iter().find(|s| s.id == last)
9466 {
9467 return s.clone();
9468 }
9469 if ok(&spec.id) {
9470 return spec;
9471 }
9472 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9473 roster
9474 .get(from + 1..)
9475 .into_iter()
9476 .flatten()
9477 .chain(roster.iter())
9478 .find(|s| ok(&s.id))
9479 .cloned()
9480 .unwrap_or(spec)
9481}
9482
9483pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9487 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9488}
9489
9490const TIMED_OUT: &str = "timed out";
9493
9494impl FailClass {
9495 fn of(out: &AgentOutcome) -> Option<Self> {
9497 match out {
9498 AgentOutcome::Ok(_) => None,
9499 AgentOutcome::Quota(_) => Some(Self::Quota),
9500 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9501 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9502 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9503 AgentOutcome::ResourceBlocked(_) => None,
9506 }
9507 }
9508
9509 fn stem_word(&self) -> &'static str {
9511 match self {
9512 Self::Quota => "quota",
9513 _ => "handover",
9514 }
9515 }
9516}
9517
9518fn failure_signature(msg: &str) -> String {
9522 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9523 if let Some(code) = line
9526 .strip_prefix("process exited with no usable output (exit code ")
9527 .and_then(|s| s.strip_suffix(')'))
9528 .and_then(|s| s.parse::<i32>().ok())
9529 {
9530 line = format!("exited with some({code}) and no usable output");
9531 } else if matches!(
9532 line.as_str(),
9533 "process exited with no usable output (no exit code reported)"
9534 | "process exited with no usable output (exit code unavailable)"
9535 ) {
9536 line = "exited with none and no usable output".to_owned();
9537 }
9538 let mut out = Vec::new();
9539 for word in line.split_whitespace() {
9540 if word.contains('/') || word.contains('\\') {
9541 out.push("<path>".to_owned());
9542 continue;
9543 }
9544 let mut w = String::new();
9545 let mut in_digits = false;
9546 for c in word.chars() {
9547 if c.is_ascii_digit() {
9548 if !in_digits {
9549 w.push('#');
9550 }
9551 in_digits = true;
9552 } else {
9553 in_digits = false;
9554 w.push(c);
9555 }
9556 }
9557 out.push(w);
9558 }
9559 out.join(" ").chars().take(120).collect()
9560}
9561
9562fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9569 match cur {
9570 FailClass::Quota | FailClass::Timeout => true,
9571 FailClass::Other(_) => prev != Some(cur),
9572 }
9573}
9574
9575fn fail_reason(out: &AgentOutcome) -> String {
9577 match out {
9578 AgentOutcome::Ok(_) => String::new(),
9579 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9580 AgentOutcome::Dropped(o) => format!(
9581 "the CLI dropped the stream ({})",
9582 o.dropped
9583 .as_ref()
9584 .map(|d| d.why.as_str())
9585 .unwrap_or("it ended without delivering its answer")
9586 ),
9587 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9588 e.lines().next().unwrap_or("").chars().take(160).collect()
9589 }
9590 }
9591}
9592
9593pub(crate) fn record_handover(
9596 state: &mut RunState,
9597 node: &str,
9598 seat: &str,
9599 from: &str,
9600 to: &str,
9601 class: &FailClass,
9602 reason: &str,
9603) {
9604 let message = if *class == FailClass::Quota {
9605 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9606 } else {
9607 format!("{seat}: handed over {from} -> {to} ({reason})")
9608 };
9609 state.event(node, message);
9610 state.handovers.push(Handover {
9611 at: Timestamp::now(),
9612 node: node.to_owned(),
9613 seat: seat.to_owned(),
9614 from: from.to_owned(),
9615 to: to.to_owned(),
9616 reason: reason.to_owned(),
9617 });
9618}
9619
9620fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9634 commands.iter().any(|c| c.exit_code.is_none())
9635}
9636
9637fn verified_noop_claim(
9650 usable: bool,
9651 commands: &[agent::CommandEvidence],
9652 text: &str,
9653) -> Option<String> {
9654 (usable && !has_unconfirmed_command(commands))
9655 .then(|| verdict::verified_noop(text))
9656 .flatten()
9657}
9658
9659fn short(commit: &str) -> String {
9660 commit.chars().take(7).collect()
9661}
9662
9663fn make_executable(path: &Path) -> Result<()> {
9664 #[cfg(unix)]
9665 {
9666 use std::os::unix::fs::PermissionsExt as _;
9667 let mut perms = std::fs::metadata(path)?.permissions();
9668 perms.set_mode(0o755);
9669 std::fs::set_permissions(path, perms)?;
9670 }
9671 #[cfg(not(unix))]
9672 {
9673 let _ = path;
9674 }
9675 Ok(())
9676}
9677
9678struct WaveCtx<'a> {
9685 run: &'a str,
9688 node: &'a str,
9690 prompts: &'a Prompts,
9691 cache: Option<&'a Path>,
9693 round: Option<usize>,
9696 carry_seats: bool,
9700 pauses: Option<(&'a Pause, &'a Pause)>,
9703}
9704
9705fn repository_wait(
9707 state: &RunState,
9708 pauses: Option<(&Pause, &Pause)>,
9709) -> crate::repo_lock::WaitPolicy {
9710 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9711 let inherited = crate::repo_lock::WaitPolicy::current();
9712 crate::repo_lock::WaitPolicy::new(
9713 Duration::from_secs(state.config.graph.timeout_resource_wait),
9714 move || {
9715 inherited.cancelled()
9716 || pauses
9717 .as_ref()
9718 .is_some_and(|(a, b)| a.parked() || b.parked())
9719 },
9720 )
9721}
9722
9723fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9724 if error.is::<crate::repo_lock::ResourceBlocked>() {
9725 state.resource_block = Some(crate::run::ResourceBlock {
9726 cause: crate::run::ResourceBlockCause::Other,
9727 node: "repository".into(),
9728 kind: if error.to_string().contains("disk pressure") {
9729 "disk pressure"
9730 } else {
9731 "repository coordination"
9732 }
9733 .into(),
9734 detail: error.to_string(),
9735 at: Timestamp::now(),
9736 });
9737 state.event("resource", error.to_string());
9738 }
9739}
9740
9741async fn coordinate_operation(
9742 state: &mut RunState,
9743 cwd: &Path,
9744 wait: &crate::repo_lock::WaitPolicy,
9745) -> Result<Option<crate::repo_lock::SeatLease>> {
9746 let result =
9747 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9748 .await;
9749 if let Err(e) = &result {
9750 record_repository_block(state, e);
9751 }
9752 result
9753}
9754
9755async fn run_one(
9757 job: SeatJob,
9758 sem: Arc<Semaphore>,
9759 ctx: &WaveCtx<'_>,
9760 state: &mut RunState,
9761 attempt: usize,
9762) -> (SeatState, AgentOutcome) {
9763 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9764 .await
9765 .pop()
9766 .expect("one job in, one result out");
9767 (seat, out)
9768}
9769
9770async fn wave(
9776 jobs: Vec<SeatJob>,
9777 sem: Arc<Semaphore>,
9778 ctx: &WaveCtx<'_>,
9779 state: &mut RunState,
9780 attempt: usize,
9781) -> Vec<(usize, SeatState, AgentOutcome)> {
9782 let WaveCtx {
9783 run,
9784 node,
9785 prompts,
9786 cache,
9787 round,
9788 carry_seats: _,
9789 pauses,
9790 } = *ctx;
9791 for job in &jobs {
9792 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9793 }
9794 if let Err(e) = state.save() {
9795 tracing::warn!("could not persist in-progress seats: {e:#}");
9800 }
9801 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9817 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9821 let cache_guard = if let Some(cache_dir) = cache {
9822 if jobs_had_a_writer {
9823 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9824 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9825 match acquire_cache_lease(
9826 state,
9827 cache_dir,
9828 &owner,
9829 resource_limit,
9830 node,
9831 node,
9832 &cancel,
9833 )
9834 .await
9835 {
9836 Ok(Some(g)) => Some(g),
9837 Ok(None) => {
9840 if let Some(block) = &mut state.resource_block {
9841 block.cause = crate::run::ResourceBlockCause::Cancelled {
9842 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9843 };
9844 }
9845 return block_wave(
9846 jobs,
9847 state,
9848 node,
9849 attempt,
9850 "parked while waiting for the shared build cache",
9851 );
9852 }
9853 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9856 Err(e) => {
9857 let why = format!("{e:#}");
9861 state.event(
9862 "resource",
9863 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9864 );
9865 return block_wave(jobs, state, node, attempt, &why);
9866 }
9867 }
9868 } else {
9869 None
9870 }
9871 } else {
9872 None
9873 };
9874 state.seats_spawning(node, attempt);
9877 if let Err(e) = state.save() {
9878 tracing::warn!("could not persist the start of a wave: {e:#}");
9879 }
9880 let slot_limit = resource_limit
9884 + jobs
9885 .iter()
9886 .map(|j| j.timeout)
9887 .max()
9888 .unwrap_or_default()
9889 .saturating_mul(jobs.len() as u32);
9890 let mut set = tokio::task::JoinSet::new();
9891 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9894 let overlay = prompts.overlay(node);
9895 let repository_wait = repository_wait(state, pauses);
9896 for (i, mut job) in jobs.into_iter().enumerate() {
9897 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9898 if cache.is_some() {
9899 job.prompt.push('\n');
9900 job.prompt
9901 .push_str(&prompt::build_cache_note(node, job.allow_write));
9902 }
9903 let sem = Arc::clone(&sem);
9904 let run = run.to_owned();
9905 let node = node.to_owned();
9906 let attachments = if node == "implement" {
9909 state.attachments.clone()
9910 } else {
9911 Vec::new()
9912 };
9913 let cache = cache
9924 .filter(|_| job.allow_write && cache_guard.is_some())
9925 .map(Path::to_path_buf);
9926 let spawned_tx = spawned_tx.clone();
9927 let repository_wait = repository_wait.clone();
9928 let coordination = state.config.coordination.clone();
9929 let fetched_base = state.fetched_base.clone();
9930 set.spawn(async move {
9931 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9935 Ok(p) => p,
9936 Err(_) => {
9937 return (
9938 i,
9939 job.seat,
9940 AgentOutcome::ResourceBlocked(format!(
9941 "no parallel slot became free within {}s",
9942 slot_limit.as_secs()
9943 )),
9944 false,
9945 );
9946 }
9947 };
9948 spawned_tx.send(job.seat.key.clone()).ok();
9949 let mut seat = job.seat;
9950 let out = repository_wait
9951 .scope(agent::invoke(
9952 &job.spec,
9953 &mut seat,
9954 &Invocation {
9955 coordination: coordination.as_ref(),
9956 fetched_base: fetched_base.as_ref(),
9957 cwd: &job.cwd,
9958 prompt: &job.prompt,
9959 timeout: job.timeout,
9960 allow_write: job.allow_write,
9961 sessions: job.sessions,
9962 artifacts: &job.artifacts,
9963 stem: &job.stem,
9964 run: &run,
9965 node: &node,
9966 cache_dir: cache.as_deref(),
9967 attachments: &attachments,
9968 writable: &[],
9969 },
9970 ))
9971 .await;
9972 let interrupted = out
9973 .as_ref()
9974 .err()
9975 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9976 let out = match out {
9977 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9978 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9979 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9987 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9988 Ok(o) => AgentOutcome::Failed(format!(
9989 "process exited with no usable output ({})",
9990 crate::notices::exit_status(o.exit_code)
9991 )),
9992 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9993 AgentOutcome::ResourceBlocked(e.to_string())
9994 }
9995 Err(e) => AgentOutcome::Failed(e.to_string()),
9996 };
9997 (i, seat, out, interrupted)
9998 });
9999 }
10000 drop(spawned_tx);
10001 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10002 loop {
10003 let joined = tokio::select! {
10004 Some(key) = spawned_rx.recv() => {
10005 state.seat_spawned(&key);
10007 if let Err(e) = state.save() {
10008 tracing::warn!("could not persist a seat's start: {e:#}");
10009 }
10010 continue;
10011 }
10012 joined = set.join_next() => joined,
10013 };
10014 let Some(joined) = joined else {
10015 break;
10016 };
10017 let (i, seat, out, interrupted) = match joined {
10018 Ok(v) => v,
10019 Err(e) => {
10023 tracing::error!("agent task panicked: {e}");
10024 continue;
10025 }
10026 };
10027 state.seat_finished(&seat.key);
10028 if interrupted
10029 && node == "implement"
10030 && let Some(label) = state
10031 .candidates
10032 .iter()
10033 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10034 .map(|candidate| candidate.label)
10035 {
10036 state.candidate_preservation.insert(
10037 label,
10038 "interrupted implementation awaiting persistence".into(),
10039 );
10040 state.interrupted_implementations.insert(label);
10041 }
10042 if let AgentOutcome::ResourceBlocked(why) = &out
10043 && state.resource_block.is_none()
10044 {
10045 state.resource_block = Some(crate::run::ResourceBlock {
10046 cause: crate::run::ResourceBlockCause::Other,
10047 node: node.to_owned(),
10048 kind: if why.contains("disk pressure") {
10049 "disk pressure"
10050 } else if why.contains("repository coordination") {
10051 "repository coordination"
10052 } else {
10053 "parallel slot"
10054 }
10055 .to_owned(),
10056 detail: why.clone(),
10057 at: Timestamp::now(),
10058 });
10059 let activity = if interrupted {
10060 "agent execution interrupted"
10061 } else {
10062 "no agent was started"
10063 };
10064 state.event("resource", format!("{node}: {activity} - {why}"));
10065 }
10066 record_jobs(state, node, round, &seat.key, &out);
10067 if let Err(e) = state.save() {
10068 tracing::warn!("could not persist a seat's completion: {e:#}");
10069 }
10070 if collected.len() <= i {
10071 collected.resize_with(i + 1, || None);
10072 }
10073 collected[i] = Some((i, seat, out));
10074 }
10075 if state
10081 .active
10082 .values()
10083 .any(|a| a.node == node && a.attempt == attempt)
10084 {
10085 state
10086 .active
10087 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10088 if let Err(e) = state.save() {
10089 tracing::warn!("could not persist the end of a wave: {e:#}");
10090 }
10091 }
10092 if let Some(cache_dir) = cache
10099 && jobs_had_a_writer
10100 {
10101 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10102 }
10103 if let Some(guard) = cache_guard {
10104 guard.release();
10105 }
10106 collected.into_iter().flatten().collect()
10107}
10108
10109fn record_jobs(
10120 state: &mut RunState,
10121 node: &str,
10122 round: Option<usize>,
10123 seat: &str,
10124 out: &AgentOutcome,
10125) {
10126 let commands: &[agent::CommandEvidence] = match out {
10127 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10128 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10129 };
10130 let checked_at = Timestamp::now();
10131 for c in commands {
10132 state.jobs.push(JobRecord {
10133 node: node.to_owned(),
10134 round,
10135 seat: seat.to_owned(),
10136 id: c.id.clone(),
10137 description: c.description.clone(),
10138 checked_at,
10139 status: match c.exit_code {
10140 Some(0) => JobStatus::Completed,
10141 Some(_) => JobStatus::Failed,
10142 None => JobStatus::Unknown,
10143 },
10144 exit_code: c.exit_code,
10145 result_summary: c.result_summary.clone(),
10146 source: c.source.clone(),
10147 });
10148 }
10149}
10150
10151fn round_is_clean(
10172 blocking: usize,
10173 e2e_ok: bool,
10174 answered: usize,
10175 expected: usize,
10176 quota_missing: usize,
10177 policy: IncompleteReviewPolicy,
10178) -> bool {
10179 if blocking != 0 || !e2e_ok {
10180 return false;
10181 }
10182 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10183 return true;
10184 }
10185 answered > 0 && expected - answered <= quota_missing
10186}
10187
10188pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10212 if max_rounds == 0
10213 || reviews.last().is_some_and(|r| {
10214 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10215 })
10216 {
10217 return Some(RunStatus::Gating);
10218 }
10219 let last = reviews.last()?;
10220 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10221 if reviews.len() < max_rounds && !stagnant {
10222 return None;
10223 }
10224 if last.incomplete() && last.blocking == 0 {
10225 return Some(RunStatus::Blocked);
10226 }
10227 if last.e2e_status() == E2eStatus::ResourceBlocked
10228 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10229 {
10230 return None;
10231 }
10232 Some(if last.e2e.iter().all(CommandOutcome::ok) {
10233 RunStatus::Gating
10234 } else {
10235 RunStatus::Blocked
10236 })
10237}
10238
10239pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10254 if nudged {
10255 (full / 4).max(Duration::from_secs(120)).min(full)
10256 } else {
10257 full
10258 }
10259}
10260
10261#[allow(clippy::too_many_arguments)]
10290async fn ask_json_wave<T>(
10291 jobs: Vec<SeatJob>,
10292 sem: Arc<Semaphore>,
10293 retries: usize,
10294 roster: &[AgentSpec],
10295 ctx: &WaveCtx<'_>,
10296 losses: &mut Vec<QuotaLoss>,
10297 state: &mut RunState,
10298 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10299) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10300where
10301 T: serde::de::DeserializeOwned + Send + 'static,
10302{
10303 ask_wave_with(
10304 jobs,
10305 sem,
10306 retries,
10307 roster,
10308 ctx,
10309 losses,
10310 state,
10311 &|text: &str| {
10312 let v = verdict::extract_json::<T>(text)?;
10313 validate(&v)?;
10314 Ok(v)
10315 },
10316 )
10317 .await
10318}
10319
10320#[allow(clippy::too_many_arguments)]
10329async fn ask_wave_with<T>(
10330 jobs: Vec<SeatJob>,
10331 sem: Arc<Semaphore>,
10332 retries: usize,
10333 roster: &[AgentSpec],
10334 ctx: &WaveCtx<'_>,
10335 losses: &mut Vec<QuotaLoss>,
10336 state: &mut RunState,
10337 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10338) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10339where
10340 T: Send + 'static,
10341{
10342 let n = jobs.len();
10343 let originals: Vec<SeatJob> = jobs;
10344 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10345 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10346 let mut nudges: Vec<usize> = vec![0; n];
10353 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10357 let mut tried: Vec<BTreeSet<String>> = specs
10358 .iter()
10359 .map(|s| BTreeSet::from([s.id.clone()]))
10360 .collect();
10361 let starts: Vec<usize> = specs
10362 .iter()
10363 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10364 .collect();
10365 let carry = ctx.carry_seats && !roster.is_empty();
10366 let carried: Vec<BTreeSet<String>> = originals
10369 .iter()
10370 .map(|j| {
10371 state
10372 .seat_history
10373 .get(&j.seat.key)
10374 .filter(|_| carry)
10375 .map(|h| h.failed.clone())
10376 .unwrap_or_default()
10377 })
10378 .collect();
10379 let mut prev: Vec<Option<FailClass>> = originals
10380 .iter()
10381 .map(|j| {
10382 state
10383 .seat_history
10384 .get(&j.seat.key)
10385 .filter(|_| carry)
10386 .and_then(|h| h.last_fail.clone())
10387 })
10388 .collect();
10389 let next_agent =
10390 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10391 let others: BTreeSet<String> = specs
10392 .iter()
10393 .enumerate()
10394 .filter(|(j, _)| *j != i)
10395 .map(|(_, s)| s.id.clone())
10396 .collect();
10397 pick_successor(
10398 roster,
10399 starts[i],
10400 tried,
10401 carry.then(|| &carried[i]),
10402 &others,
10403 )
10404 .cloned()
10405 };
10406 let mut fresh: Vec<Option<String>> = vec![None; n];
10407 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10408 let mut pending: Vec<usize> = (0..n).collect();
10409
10410 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10415 for round in 0..max_rounds {
10416 if pending.is_empty() {
10417 break;
10418 }
10419 let mut batch = Vec::with_capacity(pending.len());
10420 let mut renudged: Vec<&str> = Vec::new();
10421 for &i in &pending {
10422 let src = &originals[i];
10423 let full: &str = match &src.handover {
10428 Some(h) if specs[i].id != src.spec.id => h,
10429 _ => &src.prompt,
10430 };
10431 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10434 (
10435 full.to_owned(),
10436 src.timeout,
10437 format!("{}-{word}-{}", src.stem, specs[i].id),
10438 )
10439 } else if nudges[i] == 0 {
10440 (full.to_owned(), src.timeout, src.stem.clone())
10441 } else {
10442 renudged.push(src.seat.key.as_str());
10443 let why = done[i]
10444 .as_ref()
10445 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10446 .unwrap_or_else(|| "no parsable answer".to_owned());
10447 let nudge = prompt::nudge(&why);
10448 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10449 let prompt = if nudged {
10450 nudge
10451 } else {
10452 format!("{full}\n\n---\n\n{nudge}")
10453 };
10454 (
10455 prompt,
10456 retry_budget(src.timeout, nudged),
10457 format!("{}-retry{}", src.stem, nudges[i]),
10458 )
10459 };
10460 batch.push(SeatJob {
10461 spec: specs[i].clone(),
10462 seat: seats[i].clone(),
10463 cwd: src.cwd.clone(),
10464 prompt,
10465 timeout,
10466 allow_write: src.allow_write,
10467 sessions: src.sessions,
10468 artifacts: src.artifacts.clone(),
10469 stem,
10470 handover: None,
10471 });
10472 }
10473
10474 if !renudged.is_empty() {
10475 state.event(
10476 ctx.node,
10477 format!("retry {round}: re-asking {}", renudged.join(", ")),
10478 );
10479 }
10480 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10481 let mut still = Vec::new();
10482 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10483 seats[i] = seat;
10484 if let AgentOutcome::ResourceBlocked(why) = &out {
10487 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10488 continue;
10489 }
10490 let class = FailClass::of(&out);
10491 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10494 && nudges[i] < retries
10495 && !roster.is_empty();
10496 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10497 let next = should_hand_over(prev[i].as_ref(), &cur)
10498 .then(|| next_agent(i, &tried[i], &specs))
10499 .flatten();
10500 if carry {
10501 let h = state
10502 .seat_history
10503 .entry(originals[i].seat.key.clone())
10504 .or_default();
10505 h.failed.insert(specs[i].id.clone());
10506 h.last_fail = Some(cur.clone());
10507 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10508 h.last_ok = None;
10509 }
10510 if let Err(e) = state.save() {
10513 tracing::warn!("could not persist a seat's failure history: {e:#}");
10514 }
10515 }
10516 if let Some(next) = next {
10517 record_handover(
10518 state,
10519 ctx.node,
10520 &originals[i].seat.key,
10521 &specs[i].id,
10522 &next.id,
10523 &cur,
10524 &fail_reason(&out),
10525 );
10526 tried[i].insert(next.id.clone());
10527 prev[i] = Some(cur.clone());
10528 seats[i] =
10529 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10530 specs[i] = next;
10531 fresh[i] = Some(cur.stem_word().to_owned());
10532 nudges[i] = 0;
10533 done[i] = Some(Err(anyhow::anyhow!(
10534 "handed over after: {}",
10535 fail_reason(&out)
10536 )));
10537 still.push(i);
10538 continue;
10539 }
10540 }
10541 if carry
10542 && !nudge_first
10543 && let Some(cur) = class.clone()
10544 {
10545 let h = state
10547 .seat_history
10548 .entry(originals[i].seat.key.clone())
10549 .or_default();
10550 h.failed.insert(specs[i].id.clone());
10551 h.last_fail = Some(cur);
10552 }
10553 let parsed = match out {
10554 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10555 AgentOutcome::Quota(o) => {
10556 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10557 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10558 }
10559 AgentOutcome::Dropped(o) => {
10563 let why = o
10564 .dropped
10565 .as_ref()
10566 .map(|d| d.why.as_str())
10567 .unwrap_or("the CLI ended the stream without delivering its answer");
10568 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10569 }
10570 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10571 Err(anyhow::anyhow!(e))
10572 }
10573 };
10574 let quota = class == Some(FailClass::Quota);
10575 let failed = parsed.is_err();
10576 done[i] = Some(parsed);
10577 if carry && !failed {
10578 let h = state
10579 .seat_history
10580 .entry(originals[i].seat.key.clone())
10581 .or_default();
10582 h.failed.remove(&specs[i].id);
10583 h.last_ok = Some(specs[i].id.clone());
10584 h.last_fail = None;
10585 }
10586 let agent_failure = class.is_some()
10594 && !nudge_first
10595 && !roster.is_empty()
10596 && next_agent(i, &tried[i], &specs).is_some();
10597 if failed && !quota && !agent_failure && nudges[i] < retries {
10598 nudges[i] += 1;
10599 still.push(i);
10600 }
10601 }
10602 pending = still;
10603 }
10604
10605 for (i, q) in last_quota.into_iter().enumerate() {
10608 if let Some(reset) = q {
10609 losses.push(QuotaLoss {
10610 seat: originals[i].seat.key.clone(),
10611 node: ctx.node.to_owned(),
10612 at: Timestamp::now(),
10613 reset,
10614 });
10615 }
10616 }
10617
10618 seats
10619 .into_iter()
10620 .zip(done)
10621 .zip(nudges)
10622 .map(|((seat, res), attempts)| {
10623 (
10624 seat,
10625 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10626 attempts,
10627 )
10628 })
10629 .collect()
10630}
10631
10632fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10634 results
10635 .iter()
10636 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10637}
10638
10639fn block_wave(
10643 jobs: Vec<SeatJob>,
10644 state: &mut RunState,
10645 node: &str,
10646 attempt: usize,
10647 why: &str,
10648) -> Vec<(usize, SeatState, AgentOutcome)> {
10649 state.resource_wait = None;
10650 state
10651 .active
10652 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10653 if let Err(e) = state.save() {
10654 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10655 }
10656 jobs.into_iter()
10657 .enumerate()
10658 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10659 .collect()
10660}
10661
10662async fn acquire_cache_lease(
10675 state: &mut RunState,
10676 cache_dir: &Path,
10677 owner: &crate::cache::Owner,
10678 budget: Duration,
10679 node: &str,
10680 context: &str,
10681 cancel: &(dyn Fn() -> bool + Sync),
10682) -> Result<Option<crate::cache::Guard>> {
10683 let home = crate::run::home();
10684 let started = Instant::now();
10685 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10686 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10687 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10688 Err(e) => {
10689 state.event(
10690 "verify",
10691 format!("{context}: could not check the shared build cache: {e:#}"),
10692 );
10693 if let Err(e2) = state.save() {
10694 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10695 }
10696 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10697 }
10698 };
10699 state.event(
10700 "verify",
10701 format!(
10702 "{context}: waiting for the shared build cache at {} ({})",
10703 cache_dir.display(),
10704 busy.describe()
10705 ),
10706 );
10707 state.resource_wait = Some(crate::run::ResourceWait {
10710 node: context.to_owned(),
10711 kind: "shared build cache".to_owned(),
10712 holder: busy.describe(),
10713 since: Timestamp::now(),
10714 limit_secs: budget.as_secs(),
10715 });
10716 if let Err(e) = state.save() {
10717 tracing::warn!("could not persist a cache wait: {e:#}");
10718 }
10719 let remaining = budget.saturating_sub(started.elapsed());
10720 let waited = crate::cache::wait_for_unless(
10721 &home,
10722 cache_dir,
10723 owner,
10724 remaining,
10725 Duration::from_secs(5),
10726 cancel,
10727 )
10728 .await;
10729 state.resource_wait = None;
10730 let cause = match &waited {
10731 Ok(Some(_)) => None,
10732 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10733 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10734 Some(crate::run::ResourceBlockCause::Timeout {
10735 waited_secs: started.elapsed().as_secs(),
10736 limit_secs: budget.as_secs(),
10737 })
10738 }
10739 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10740 };
10741 if let Some(cause) = cause {
10742 let detail = match &waited {
10743 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10744 Err(error) => format!("{error:#}"),
10745 };
10746 state.resource_block = Some(crate::run::ResourceBlock {
10747 cause,
10748 node: node.to_owned(),
10749 kind: "shared build cache".into(),
10750 detail,
10751 at: Timestamp::now(),
10752 });
10753 state.event(
10754 "resource",
10755 state.resource_block.as_ref().unwrap().description(),
10756 );
10757 }
10758 if let Err(error) = state.save() {
10759 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10760 }
10761 waited
10762}
10763
10764#[derive(Debug)]
10769struct CacheCheckError(String);
10770
10771impl std::fmt::Display for CacheCheckError {
10772 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10773 f.write_str(&self.0)
10774 }
10775}
10776
10777impl std::error::Error for CacheCheckError {}
10778
10779fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10781 state
10782 .resource_block
10783 .get_or_insert_with(|| crate::run::ResourceBlock {
10784 cause: crate::run::ResourceBlockCause::Other,
10785 node: node.to_owned(),
10786 kind: "shared build cache".into(),
10787 detail: format!("{error:#}"),
10788 at: Timestamp::now(),
10789 });
10790 if let Err(error) = state.save() {
10791 tracing::warn!("could not persist cache resource block: {error:#}");
10792 }
10793}
10794
10795#[allow(clippy::too_many_arguments)]
10816async fn with_cache_lease<'s, F, Fut>(
10817 state: &'s mut RunState,
10818 cache_dir: Option<&Path>,
10819 node: &str,
10820 seat: &str,
10821 worktree: &Path,
10822 head: &str,
10823 budget: Duration,
10824 context: &str,
10825 wait: &crate::repo_lock::WaitPolicy,
10826 body: F,
10827) -> Result<(Vec<CommandOutcome>, bool)>
10828where
10829 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10830 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10831{
10832 let started = Instant::now();
10835 let home = crate::run::home();
10836 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10837 let guard = if let Some(cache_dir) = cache_dir {
10838 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10839 match acquire_cache_lease(
10840 state,
10841 cache_dir,
10842 &owner,
10843 resource_limit,
10844 node,
10845 context,
10846 &|| wait.cancelled(),
10847 )
10848 .await
10849 .and_then(|g| {
10850 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10851 }) {
10852 Ok(g) => Some(g),
10853 Err(e) => {
10854 record_cache_error(state, node, &e);
10855 return Ok((
10856 vec![CommandOutcome {
10857 command: "(waiting for the shared build cache)".into(),
10858 code: None,
10859 output_tail: e.to_string(),
10860 duration_ms: started.elapsed().as_millis() as u64,
10861 resource_blocked: true,
10862 }],
10863 false,
10864 ));
10865 }
10866 }
10867 } else {
10868 None
10869 };
10870 let _coordination = coordinate_operation(state, worktree, wait).await?;
10871 if let Some(cache_dir) = cache_dir {
10872 let identity = crate::cache::Identity::new(worktree, head);
10873 if let Err(e) = crate::cache::ensure_fresh(
10874 &home,
10875 cache_dir,
10876 &identity,
10877 &state.config.verify.cargo_manifests(),
10878 ) {
10879 record_cache_error(state, node, &e);
10880 state.event(
10881 "verify",
10882 format!(
10883 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10884 worktree.display(),
10885 short(head)
10886 ),
10887 );
10888 if let Some(guard) = guard {
10889 guard.release();
10890 }
10891 return Ok((
10892 vec![CommandOutcome {
10893 command: "(confirming the shared build cache is fresh)".into(),
10894 code: None,
10895 output_tail: e.to_string(),
10896 duration_ms: started.elapsed().as_millis() as u64,
10897 resource_blocked: true,
10898 }],
10899 false,
10900 ));
10901 }
10902 }
10903 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10904 if !timed_out_pids.is_empty() {
10905 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10906 }
10907 drop(_coordination);
10910 if let Some(guard) = guard {
10911 guard.release();
10912 }
10913 Ok((outcomes, retried))
10914}
10915
10916async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10928 wait_for_pids_with(
10929 pids,
10930 crate::proc::pid_alive,
10931 LEASE_RELEASE_POLL,
10932 LEASE_RELEASE_MAX_WAIT,
10933 )
10934 .await;
10935}
10936
10937async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10943 pids: &[u32],
10944 alive: F,
10945 poll: Duration,
10946 max_wait: Duration,
10947) {
10948 let deadline = Instant::now() + max_wait;
10949 loop {
10950 if pids.iter().all(|&pid| !alive(pid)) {
10951 return;
10952 }
10953 if Instant::now() >= deadline {
10954 return;
10955 }
10956 tokio::time::sleep(poll).await;
10957 }
10958}
10959
10960fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10967 outcomes.iter().any(|o| o.resource_blocked)
10968}
10969
10970pub(crate) enum GateFix {
10972 Retry,
10974 Stop,
10977 Defer,
10980}
10981
10982fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10990 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10991 red.peek().is_some()
10992 && red.all(|o| {
10993 !o.resource_blocked
10994 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10995 && !o.output_tail.trim().is_empty()
10996 })
10997}
10998
10999fn e2e_outcome_label(o: &CommandOutcome) -> String {
11003 if o.ok() {
11004 return "pass".to_owned();
11005 }
11006 let reason = if o.build_failed() {
11007 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11008 } else {
11009 format!("FAIL ({:?})", o.code)
11010 };
11011 format!(
11012 "{reason}\n{}",
11013 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11014 )
11015}
11016
11017async fn run_e2e_with_retry(
11025 state: &mut RunState,
11026 shell: &[String],
11027 commands: &[String],
11028 worktree: &Path,
11029 timeout: Duration,
11030 context: &str,
11031) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11032 let (mut e2e, mut timed_out_pids) = run_commands(
11033 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11034 )
11035 .await;
11036 for o in &e2e {
11037 state.event(
11038 "verify",
11039 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11040 );
11041 }
11042 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11046 if verify_retried {
11047 state.event(
11048 "verify",
11049 format!(
11050 "{context}: verify could not build/link, not a test result — retrying once \
11051 before concluding"
11052 ),
11053 );
11054 let retried = run_commands(
11055 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11056 )
11057 .await;
11058 e2e = retried.0;
11059 timed_out_pids.extend(retried.1);
11062 for o in &e2e {
11063 state.event(
11064 "verify",
11065 format!(
11066 "{context}: retry `{}` -> {}",
11067 o.command,
11068 e2e_outcome_label(o)
11069 ),
11070 );
11071 }
11072 }
11073 (e2e, verify_retried, timed_out_pids)
11074}
11075
11076#[allow(clippy::too_many_arguments)]
11092async fn run_commands(
11093 state: &mut RunState,
11094 node: &str,
11095 task: &str,
11096 attempt: usize,
11097 shell: &[String],
11098 commands: &[String],
11099 cwd: &Path,
11100 timeout: Duration,
11101) -> (Vec<CommandOutcome>, Vec<u32>) {
11102 if commands.is_empty() {
11103 return (Vec::new(), Vec::new());
11108 }
11109 let mut out = Vec::new();
11110 let mut timed_out_pids = Vec::new();
11111 let total = commands.len();
11112 for (idx, command) in commands.iter().enumerate() {
11113 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11114 if let Err(e) = state.save() {
11115 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11116 }
11117 let started = Instant::now();
11118 let mut cmd = tokio::process::Command::new(&shell[0]);
11119 cmd.quiet();
11120 for key in crate::worker::INVOCATION_ENV {
11124 cmd.env_remove(key);
11125 }
11126 cmd.args(&shell[1..])
11127 .arg(command)
11128 .current_dir(cwd)
11129 .stdin(std::process::Stdio::null())
11130 .stdout(std::process::Stdio::piped())
11131 .stderr(std::process::Stdio::piped())
11132 .kill_on_drop(true);
11133 let mut outputs = Vec::new();
11134 let protected = (|| -> Result<()> {
11135 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
11136 outputs.push(guard);
11137 }
11138 if let Some(cache) = state.config.cache_dir() {
11139 outputs.push(crate::retention::OutputLease::cache(
11140 &crate::run::home(),
11141 &cache,
11142 false,
11143 )?);
11144 }
11145 Ok(())
11146 })();
11147 if let Err(e) = protected {
11148 out.push(CommandOutcome {
11149 command: command.clone(),
11150 code: None,
11151 output_tail: format!("output ownership unavailable: {e:#}"),
11152 duration_ms: 0,
11153 resource_blocked: true,
11154 });
11155 continue;
11156 }
11157 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
11158 let (code, body) = match spawned {
11159 Ok(mut running) => {
11160 let pid = running.child().id();
11161 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
11162 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
11163 let mut paths = vec![cwd.to_owned()];
11164 if let Some(cache) = state.config.cache_dir() {
11165 paths.push(cache);
11166 }
11167 let mut pressure = None;
11168 let status = tokio::select! {
11169 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
11170 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
11171 pressure = Some(reason); None
11172 }
11173 };
11174 let (code, failure) = match status {
11175 Some(Ok(Ok(status))) => (
11176 status.code(),
11177 status
11178 .code()
11179 .is_none()
11180 .then(|| format!("command terminated: {status}")),
11181 ),
11182 Some(Ok(Err(error))) => {
11183 (None, Some(format!("failed waiting for command: {error}")))
11184 }
11185 Some(Err(_)) => {
11186 if !running.coordinated()
11190 && let Some(pid) = pid
11191 {
11192 timed_out_pids.push(pid);
11193 }
11194 (
11195 None,
11196 Some(format!("timed out after {}s", timeout.as_secs())),
11197 )
11198 }
11199 None => (None, None),
11200 };
11201 running.finish().await;
11202 if let Some(reason) = pressure {
11203 state.resource_block = Some(crate::run::ResourceBlock {
11204 cause: crate::run::ResourceBlockCause::Other,
11205 node: node.to_owned(),
11206 kind: "disk pressure".to_owned(),
11207 detail: reason.clone(),
11208 at: Timestamp::now(),
11209 });
11210 out.push(CommandOutcome {
11211 command: command.clone(),
11212 code: None,
11213 output_tail: reason,
11214 duration_ms: started.elapsed().as_millis() as u64,
11215 resource_blocked: true,
11216 });
11217 break;
11218 }
11219 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11220 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11221 if let Some(failure) = failure {
11222 body.push('\n');
11223 body.push_str(&failure);
11224 }
11225 (code, body)
11226 }
11227 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11228 };
11229 let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
11230 if !body.is_empty() {
11231 match crate::run::write_verification_artifact(state, task, &body) {
11235 Ok(name) if summary != body => {
11236 summary = format!("[full output: artifacts/{name}]\n{summary}");
11237 }
11238 Ok(_) => {}
11239 Err(e) => {
11240 let warning = format!("could not save the full {task} output: {e:#}");
11241 tracing::warn!("{warning}");
11242 state.event(node, warning);
11243 if summary != body {
11244 summary = format!("[full output could not be saved: {e:#}]\n{summary}");
11245 }
11246 }
11247 }
11248 }
11249 out.push(CommandOutcome {
11250 command: command.clone(),
11251 code,
11252 output_tail: summary,
11253 duration_ms: started.elapsed().as_millis() as u64,
11254 resource_blocked: false,
11255 });
11256 }
11257 state.task_finished(task);
11258 if let Err(e) = state.save() {
11259 tracing::warn!("could not persist the end of {task}: {e:#}");
11260 }
11261 (out, timed_out_pids)
11262}
11263
11264fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11276 let repo = repo.display();
11277 match style {
11278 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11279 MergeStyle::Squash => {
11280 let subject = message
11283 .lines()
11284 .next()
11285 .unwrap_or(branch)
11286 .replace(['\\', '"', '$', '`'], "");
11287 format!(
11288 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11289 )
11290 }
11291 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11292 }
11293}
11294
11295const PR_TITLE_MAX: usize = 240;
11306
11307fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11318 let line = queue::first_line(text)?;
11319 let chars: Vec<char> = line.chars().collect();
11320 let (kept, mark) = if chars.len() <= max {
11321 (line.to_owned(), "")
11322 } else if let Some(end) = sentence_end(&chars, max) {
11323 (chars[..end].iter().collect::<String>(), "")
11324 } else {
11325 let room = max.saturating_sub(3);
11326 let cut = if chars[room].is_whitespace() {
11329 room
11330 } else {
11331 chars[..room]
11332 .iter()
11333 .rposition(|c| c.is_whitespace())
11334 .unwrap_or(room)
11335 };
11336 let head: String = chars[..cut].iter().collect();
11337 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11338 (head.to_owned(), "...")
11339 };
11340 if kept.is_empty() {
11341 return None;
11342 }
11343 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11344 return None;
11345 }
11346 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11347 return None;
11348 }
11349 Some(format!("{kept}{mark}"))
11350}
11351
11352fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11356 const MIN: usize = 20;
11357 for i in MIN..max.min(chars.len()) {
11358 if !matches!(chars[i], '.' | '!' | '?') {
11359 continue;
11360 }
11361 let Some(&next) = chars.get(i + 1) else {
11362 continue;
11363 };
11364 if !next.is_whitespace() {
11365 continue;
11366 }
11367 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11368 if after.is_some_and(|c| c.is_ascii_lowercase()) {
11369 continue;
11370 }
11371 let word: String = chars[..i]
11372 .iter()
11373 .rev()
11374 .take_while(|c| !c.is_whitespace())
11375 .collect::<Vec<_>>()
11376 .into_iter()
11377 .rev()
11378 .collect();
11379 let word = word.to_ascii_lowercase();
11380 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11381 continue;
11382 }
11383 let end = chars[..i]
11384 .iter()
11385 .rposition(|c| !c.is_whitespace())
11386 .map_or(i, |p| p + 1);
11387 return Some(end);
11388 }
11389 None
11390}
11391
11392struct PrMessage {
11397 title: String,
11398 body: String,
11399}
11400
11401impl PrMessage {
11402 fn commit_message(&self) -> String {
11406 format!("{}\n\n{}", self.title, self.body)
11407 }
11408}
11409
11410fn title_marker(line: &str) -> Option<&str> {
11412 let line = line.trim();
11413 let head = line.get(..6)?;
11414 head.eq_ignore_ascii_case("title:")
11415 .then(|| line[6..].trim())
11416}
11417
11418fn summary_title(summary: &str) -> Option<String> {
11423 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11424 let raw = title_marker(first)?;
11425 if raw.is_empty() {
11426 return None;
11427 }
11428 let title = queue::title_from(raw, PR_TITLE_MAX);
11429 let lower = title.to_ascii_lowercase();
11430 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11431 return None;
11432 }
11433 Some(title)
11434}
11435
11436const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11438
11439pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11441
11442pub(crate) fn is_review_run(state: &RunState) -> bool {
11446 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11447}
11448
11449fn review_title(state: &RunState) -> Option<String> {
11454 english_subject(
11455 state.reviewed_commits.as_ref()?.first()?,
11456 state.config.graph.github_english(),
11457 )
11458}
11459
11460fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11463 let raw = raw.trim();
11464 if raw.is_empty() {
11465 return None;
11466 }
11467 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11468 return None;
11469 }
11470 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11471 return None;
11472 }
11473 let title = queue::title_from(raw, PR_TITLE_MAX);
11474 let lower = title.to_ascii_lowercase();
11475 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11476 return None;
11477 }
11478 Some(title)
11479}
11480
11481#[derive(Debug, Clone, PartialEq, Eq)]
11484struct BranchFacts {
11485 commits: Vec<(String, String)>,
11487 stat: String,
11489}
11490
11491const STAT_FILE_LINES: usize = 25;
11493const COMMITS_MAX_CHARS: usize = 20_000;
11495
11496async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11499 let commits = git::commit_log(repo, base, branch).await.ok()?;
11500 if commits.is_empty() {
11501 return None;
11502 }
11503 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11504 let lines: Vec<&str> = stat.lines().collect();
11505 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11506 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11507 let more = format!(" ... {omitted} more file(s)");
11508 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11509 kept.push(&more);
11510 kept.push(lines[lines.len() - 1]);
11511 kept.join("\n")
11512 } else {
11513 lines.join("\n")
11514 };
11515 Some(BranchFacts { commits, stat })
11516}
11517
11518fn markdown_safe(text: &str) -> String {
11521 text.replace("</details>", "</details>")
11522 .replace("\x60\x60\x60", "~~~")
11523}
11524
11525fn neutral_title(state: &RunState, winner: char) -> String {
11526 format!(
11527 "chore: land candidate {} of run {}",
11528 winner.to_ascii_uppercase(),
11529 state.id
11530 )
11531}
11532
11533pub(crate) async fn repair_landing_gate(
11536 state: &mut RunState,
11537 winner: &Candidate,
11538 outcomes: &[CommandOutcome],
11539) -> Result<GateFix> {
11540 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11541 return Ok(GateFix::Stop);
11542 }
11543 let mut runner = Runner {
11544 roles: state.config.resolve_roles()?,
11545 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11546 state: state.clone(),
11547 pause: Pause::new(),
11548 interrupt: Pause::new(),
11549 };
11550 let result = runner.gate_fix_round(winner, outcomes).await;
11551 *state = runner.state;
11552 result
11553}
11554
11555pub(crate) async fn verify_current_head(
11558 state: &mut RunState,
11559 winner: &Candidate,
11560 wait: &crate::repo_lock::WaitPolicy,
11561) -> Result<bool> {
11562 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11563 if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
11565 state
11566 .verification_history
11567 .push(crate::run::VerificationEvidence {
11568 superseded_by: Some(head.clone()),
11569 archived_at: Some(Timestamp::now()),
11570 fixes_spent: Some(state.gate_fixes.len()),
11571 gate_head: None,
11572 gate_ran: false,
11573 gate: Vec::new(),
11574 e2e: state.head_e2e.take(),
11575 });
11576 }
11577 if let Some(e) = &state.head_e2e
11578 && e.head == head
11579 {
11580 if e.passed {
11581 return Ok(true);
11582 }
11583 if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
11584 state.status = RunStatus::Blocked;
11585 state.save()?;
11586 return Ok(false);
11587 }
11588 } else if state.reviews.last().is_some_and(|r| {
11589 r.verified_head.as_deref() == Some(head.as_str())
11590 && (r.e2e_status() == E2eStatus::Passed
11591 || (r.e2e_status() == E2eStatus::NotConfigured
11592 && state.config.verify.e2e.is_empty()))
11593 }) || state
11594 .gate_repair_e2e
11595 .as_ref()
11596 .is_some_and(|e| e.head == head && e.passed)
11597 {
11598 return Ok(true);
11599 }
11600 if let Some(e) = &state.gate_repair_e2e
11601 && e.head == head
11602 && !e.passed
11603 && !e.outcomes.is_empty()
11604 && !verify_inconclusive(&e.outcomes)
11605 {
11606 state.status = RunStatus::Blocked;
11607 state.save()?;
11608 return Ok(false);
11609 }
11610 state.head_e2e = Some(crate::run::GateRepairE2e {
11611 head: head.clone(),
11612 passed: false,
11613 outcomes: Vec::new(),
11614 });
11615 state.save()?;
11616 let commands = state.config.verify.e2e.clone();
11617 let shell = state.config.shell();
11618 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11619 let cache_dir = state.config.cache_dir();
11620 let outcomes = if commands.is_empty() {
11621 Vec::new()
11622 } else {
11623 let (outcomes, _) = with_cache_lease(
11624 state,
11625 cache_dir.as_deref(),
11626 "e2e",
11627 "e2e",
11628 &winner.worktree,
11629 &head,
11630 timeout,
11631 "current head e2e",
11632 wait,
11633 |state, budget| async move {
11634 run_e2e_with_retry(
11635 state,
11636 &shell,
11637 &commands,
11638 &winner.worktree,
11639 budget,
11640 "current head e2e",
11641 )
11642 .await
11643 },
11644 )
11645 .await?;
11646 outcomes
11647 };
11648 let unresolved = verify_inconclusive(&outcomes);
11649 let passed = !unresolved
11650 && outcomes.iter().all(CommandOutcome::ok)
11651 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11652 state.head_e2e = Some(crate::run::GateRepairE2e {
11653 head: head.clone(),
11654 passed,
11655 outcomes,
11656 });
11657 if !passed && !unresolved {
11658 state.status = RunStatus::Blocked;
11659 state.event(
11660 "e2e",
11661 format!("verify.e2e failed on head {head}; nothing published"),
11662 );
11663 }
11664 state.save()?;
11665 Ok(passed)
11666}
11667
11668pub(crate) async fn check_gate_repair_e2e(
11671 state: &mut RunState,
11672 winner: &Candidate,
11673 wait: &crate::repo_lock::WaitPolicy,
11674) -> Result<GateFix> {
11675 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11676 return Ok(GateFix::Retry);
11677 }
11678 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11679 if state
11680 .gate_repair_e2e
11681 .as_ref()
11682 .is_some_and(|r| r.head == head && r.passed)
11683 {
11684 return Ok(GateFix::Retry);
11685 }
11686 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11687 head: head.clone(),
11688 passed: false,
11689 outcomes: Vec::new(),
11690 });
11691 state.save()?;
11692 let commands = state.config.verify.e2e.clone();
11693 let outcomes = if commands.is_empty() {
11694 Vec::new()
11695 } else {
11696 let shell = state.config.shell();
11697 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11698 let cache_dir = state.config.cache_dir();
11699 let (outcomes, _) = with_cache_lease(
11700 state,
11701 cache_dir.as_deref(),
11702 "e2e",
11703 "e2e",
11704 &winner.worktree,
11705 &head,
11706 timeout,
11707 "gate repair e2e",
11708 wait,
11709 |state, budget| async move {
11710 run_e2e_with_retry(
11711 state,
11712 &shell,
11713 &commands,
11714 &winner.worktree,
11715 budget,
11716 "gate repair e2e",
11717 )
11718 .await
11719 },
11720 )
11721 .await?;
11722 outcomes
11723 };
11724 let deferred = verify_inconclusive(&outcomes);
11725 let passed = outcomes.iter().all(CommandOutcome::ok)
11726 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11727 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11728 head,
11729 passed,
11730 outcomes,
11731 });
11732 state.save()?;
11733 if deferred {
11734 Ok(GateFix::Defer)
11735 } else if !passed {
11736 state.event(
11737 "gate",
11738 "verify.e2e failed after gate repair; nothing published",
11739 );
11740 Ok(GateFix::Stop)
11741 } else {
11742 Ok(GateFix::Retry)
11743 }
11744}
11745
11746pub(crate) async fn run_landing_gate(
11748 state: &mut RunState,
11749 winner: &Candidate,
11750 wait: &crate::repo_lock::WaitPolicy,
11751) -> Result<Vec<CommandOutcome>> {
11752 let shell = state.config.shell();
11753 let gate_commands = state.config.verify.gate.clone();
11754 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11755 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11756 state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
11757 Vec::new()
11758 } else {
11759 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11760 let cache_dir = state.config.cache_dir();
11761 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11762 let (outcomes, _) = with_cache_lease(
11763 state,
11764 cache_dir.as_deref(),
11765 "gate",
11766 "gate",
11767 &winner.worktree,
11768 &head,
11769 timeout,
11770 "final gate",
11771 wait,
11772 |state, budget| {
11773 let shell = shell.clone();
11774 let gate_commands = gate_commands.clone();
11775 let winner = winner.clone();
11776 async move {
11777 Runner::run_pre_gate(state, &winner).await;
11778 state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
11780 let (outcomes, timed_out_pids) = run_commands(
11781 state,
11782 "gate",
11783 "gate",
11784 0,
11785 &shell,
11786 &gate_commands,
11787 &winner.worktree,
11788 budget,
11789 )
11790 .await;
11791 (outcomes, false, timed_out_pids)
11792 }
11793 },
11794 )
11795 .await?;
11796 outcomes
11797 };
11798 let current = git::rev_parse(&winner.worktree, "HEAD").await?;
11799 if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
11800 let mut outcomes = outcomes;
11801 outcomes.push(CommandOutcome {
11802 command: "gate head stability".into(), code: None,
11803 output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
11804 duration_ms: 0, resource_blocked: false,
11805 });
11806 return Ok(outcomes);
11807 }
11808 if outcomes.is_empty() {
11809 state.event("gate", "no gate commands configured; not tested");
11814 }
11815 for o in &outcomes {
11816 state.event(
11817 "gate",
11818 format!(
11819 "`{}` -> {}",
11820 o.command,
11821 if o.ok() {
11822 "pass".to_owned()
11823 } else {
11824 format!(
11825 "FAIL ({:?})\n{}",
11826 o.code,
11827 summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11828 )
11829 }
11830 ),
11831 );
11832 }
11833 Ok(outcomes)
11834}
11835
11836pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11842 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11843 ""
11844 } else {
11845 pr_title
11846 }
11847}
11848
11849pub fn landing_subject_source(state: &RunState) -> String {
11853 if is_review_run(state) {
11854 let winner = state.candidates.first().map_or('A', |c| c.label);
11855 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11856 }
11857 state.instruction.clone()
11858}
11859
11860fn summary_without_title(summary: &str) -> String {
11863 let mut lines = summary.trim().lines().peekable();
11864 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11865 lines.next();
11866 }
11867 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11868}
11869
11870#[cfg(test)]
11882fn pr_message(state: &RunState, winner: char) -> PrMessage {
11883 pr_message_with(state, winner, None)
11884}
11885
11886#[cfg(test)]
11889fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11890 let raw = pr_message_raw(
11891 state,
11892 winner,
11893 facts,
11894 crate::pr_outcome::FinalHead::Unreported,
11895 ChangeNote::NotAsked,
11896 );
11897 let id = crate::scrub::Identity::current();
11898 PrMessage {
11899 title: crate::scrub::scrub(&raw.title, &id),
11900 body: crate::scrub::scrub(&raw.body, &id),
11901 }
11902}
11903
11904#[derive(Debug, Clone, Copy)]
11906enum ChangeNote<'a> {
11907 NotAsked,
11909 Summary(&'a crate::run::ChangeSummary),
11911 Unavailable(&'a str),
11913}
11914
11915fn pr_message_raw(
11916 state: &RunState,
11917 winner: char,
11918 facts: Option<&BranchFacts>,
11919 head: crate::pr_outcome::FinalHead<'_>,
11920 change: ChangeNote<'_>,
11921) -> PrMessage {
11922 if !matches!(change, ChangeNote::NotAsked) {
11923 let (title, mut body) = match change {
11924 ChangeNote::Summary(c) => (
11925 c.title.clone(),
11926 format!("## Summary\n\n{}\n\n", c.body.trim()),
11927 ),
11928 ChangeNote::Unavailable(why) => (
11929 neutral_title(state, winner),
11930 format!(
11931 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11932 ),
11933 ),
11934 ChangeNote::NotAsked => unreachable!(),
11935 };
11936 body.push_str(&public_pr_evidence(state, head));
11937 body.push_str(&format!(
11938 "\n---\nmagi:run/{} magi:candidate-{}\n",
11939 state.id,
11940 winner.to_ascii_lowercase()
11941 ));
11942 return PrMessage { title, body };
11943 }
11944 pr_preview(state, winner, facts, head)
11945}
11946
11947fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11950 let mut body = crate::pr_outcome::review_rounds(state);
11951 body.push_str(&crate::pr_outcome::final_verification(state, head));
11952 let open = state.last_round_findings();
11953 if !open.is_empty() {
11954 body.push_str("## Open review findings\n\n");
11955 for f in open {
11956 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11957 }
11958 body.push('\n');
11959 }
11960 if let Some(c) = state.change_summary.as_ref()
11961 && head == crate::pr_outcome::FinalHead::At(&c.head)
11962 {
11963 body.push_str(&format!(
11964 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11965 c.base, c.head
11966 ));
11967 }
11968 body
11969}
11970
11971fn pr_preview(
11973 state: &RunState,
11974 winner: char,
11975 facts: Option<&BranchFacts>,
11976 head: crate::pr_outcome::FinalHead<'_>,
11977) -> PrMessage {
11978 let summary = state
11979 .candidates
11980 .iter()
11981 .find(|c| c.label == winner)
11982 .map(|c| c.summary.as_str())
11983 .unwrap_or_default();
11984 let review = is_review_run(state);
11987 let title = if review {
11988 facts
11989 .and_then(|f| {
11990 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11991 })
11992 .or_else(|| review_title(state))
11993 .or_else(|| {
11994 state
11995 .candidates
11996 .iter()
11997 .find(|c| c.label == winner)
11998 .filter(|c| !c.branch.starts_with("magi/"))
11999 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12000 })
12001 .unwrap_or_else(|| neutral_title(state, winner))
12002 } else {
12003 summary_title(summary).unwrap_or_else(|| {
12004 english_title(
12005 &state.instruction,
12006 PR_TITLE_MAX,
12007 state.config.graph.github_english(),
12008 )
12009 .unwrap_or_else(|| neutral_title(state, winner))
12010 })
12011 };
12012
12013 let mut body = String::new();
12014 let what = summary_without_title(summary);
12015 if !what.is_empty() {
12016 body.push_str("## Summary\n\n");
12017 body.push_str(&what);
12018 body.push_str("\n\n");
12019 }
12020
12021 body.push_str(&crate::pr_outcome::review_rounds(state));
12023 body.push_str(&crate::pr_outcome::final_verification(state, head));
12024 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12025
12026 let open = state.open_findings();
12027 if !open.is_empty() {
12028 body.push_str("## Open review findings\n\n");
12029 for f in &open {
12030 body.push_str(&format!(
12031 "- `{}` [{:?}] {}\n",
12032 f.id, f.severity, "Context retained in local review records."
12033 ));
12034 }
12035 body.push('\n');
12036 }
12037
12038 if let Some(fix) = fix
12039 && !fix.rejected.is_empty()
12040 {
12041 body.push_str("## Declined by the fixer\n\n");
12042 for r in &fix.rejected {
12043 body.push_str(&format!(
12044 "- `{}`: {}\n",
12045 r.id, "Reason retained in local review records."
12046 ));
12047 }
12048 body.push('\n');
12049 }
12050
12051 if review {
12052 body.push_str("## Commits under review\n\n");
12054 if let Some(facts) = facts {
12055 let mut left = COMMITS_MAX_CHARS;
12056 for (i, (subject, text)) in facts.commits.iter().enumerate() {
12057 let mut entry = format!("- {}\n", markdown_safe(subject));
12058 for l in markdown_safe(text).lines() {
12059 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
12060 }
12061 if left == 0 {
12062 body.push_str(&format!(
12063 "- ... {} more commit(s)\n",
12064 facts.commits.len() - i
12065 ));
12066 break;
12067 }
12068 if entry.len() > left {
12069 let mut end = left;
12072 while !entry.is_char_boundary(end) {
12073 end -= 1;
12074 }
12075 entry.truncate(end);
12076 entry.push_str("\n ... (truncated)\n");
12077 left = 0;
12078 } else {
12079 left -= entry.len();
12080 }
12081 body.push_str(&entry);
12082 }
12083 if !facts.stat.trim().is_empty() {
12084 body.push_str(&format!(
12085 "\n## Diff stat\n\n```\n{}\n```\n",
12086 markdown_safe(facts.stat.trim())
12087 ));
12088 }
12089 } else {
12090 match &state.reviewed_commits {
12091 Some(subjects) => {
12092 for s in subjects {
12093 body.push_str(&format!("- {}\n", s.trim()));
12094 }
12095 }
12096 None => {
12097 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
12100 body.push_str(rest.trim());
12101 body.push('\n');
12102 }
12103 }
12104 }
12105 } else {
12106 let task = state.instruction.trim();
12107 let task = if task.is_empty() {
12108 "(empty task)"
12109 } else {
12110 task
12111 };
12112 body.push_str(&format!(
12113 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
12114 task.replace("</details>", "</details>")
12115 ));
12116 }
12117
12118 body.push_str(&format!(
12119 "\n---\nmagi:run/{} magi:candidate-{}\n",
12120 state.id,
12121 winner.to_ascii_lowercase()
12122 ));
12123
12124 PrMessage { title, body }
12125}
12126
12127fn seeded_instruction(state: &RunState) -> String {
12130 let mut inputs = Vec::new();
12131 if let Some(start) = &state.sources.start_from {
12132 inputs.push(format!(
12133 "Start from `{}` at `{}` (including ancestry).",
12134 start.token, start.sha
12135 ));
12136 }
12137 for pick in &state.sources.pick_commits {
12138 inputs.push(format!(
12139 "Pick `{}` at `{}` (single commit).",
12140 pick.token, pick.sha
12141 ));
12142 }
12143 if inputs.is_empty() {
12144 state.instruction.clone()
12145 } else {
12146 format!(
12147 "{}\n\n# Explicit source inputs already prepared\n\n{}",
12148 state.instruction,
12149 inputs.join("\n")
12150 )
12151 }
12152}
12153
12154async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
12158 let base = &state.base_branch;
12159 let mut against = base.clone();
12160 if mode == MergeMode::Pr {
12161 let remote = &state.config.merge.remote;
12165 let tracking = format!("{remote}/{base}");
12166 let fetched = git::fetch(repo, remote, base).await;
12167 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
12168 against = tracking;
12169 } else {
12170 return false;
12171 }
12172 }
12173 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
12174}
12175
12176fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
12179 let mut detail = format!(
12180 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
12181 nothing to open a pull request for"
12182 );
12183 detail.push_str(
12184 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
12185 );
12186 detail
12187}
12188
12189#[derive(Debug, PartialEq, Eq)]
12192enum PrPlan {
12193 Create,
12194 Adopt { url: String, title: String },
12195 Stop(String),
12196}
12197
12198fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
12201 match found {
12202 Ok(land::OpenPr::None) => PrPlan::Create,
12203 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
12204 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
12205 "several open pull requests exist for this branch, not picking one: {}",
12206 urls.join(" ")
12207 )),
12208 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
12209 }
12210}
12211
12212pub(crate) async fn gh_pr_create(
12214 cwd: &Path,
12215 base: &str,
12216 head: &str,
12217 title: &str,
12218 body: &str,
12219) -> Result<String> {
12220 let out = tokio::process::Command::new("gh")
12221 .args([
12222 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
12223 ])
12224 .current_dir(cwd)
12225 .quiet()
12226 .stdin(std::process::Stdio::null())
12227 .output()
12228 .await
12229 .context("spawn gh")?;
12230 if out.status.success() {
12231 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
12232 } else {
12233 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
12234 }
12235}
12236
12237async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
12239 match std::fs::symlink_metadata(path) {
12240 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
12241 git::worktree_remove_clean(repo, path).await?;
12244 return Ok(());
12245 }
12246 Err(e) => return Err(e.into()),
12247 Ok(_) => {}
12248 }
12249 if !git::is_clean(path).await? {
12250 bail!(
12251 "worktree has uncommitted work; refusing removal: {}",
12252 path.display()
12253 );
12254 }
12255 if !git::worktree_remove_clean(repo, path).await? {
12256 bail!("worktree removal refused: {}", path.display());
12257 }
12258 Ok(())
12259}
12260
12261pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
12270 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
12271 bail!("refusing fold: {reason}");
12272 }
12273 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
12274 bail!("candidate preservation incomplete; refusing cleanup");
12275 }
12276 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
12280 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
12281 let _driver = crate::driver::maintenance(home, &state.id)?;
12282 let mut output_guards = Vec::new();
12283 for entry in std::fs::read_dir(state.worktree_root())
12284 .into_iter()
12285 .flatten()
12286 .flatten()
12287 {
12288 if entry.path().join(".git").is_file() {
12289 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
12290 }
12291 }
12292 let repo = state.repo.clone();
12293 let wait = repository_wait(state, None);
12296 let _coordination = match coordinate_operation(state, &repo, &wait).await {
12297 Ok(lease) => lease,
12298 Err(e) => {
12299 state.save_under(home)?;
12300 return Err(e);
12301 }
12302 };
12303 let _source_outputs = if state.status == RunStatus::Merged
12306 && state.merge.as_ref().is_some_and(|merge| merge.ok)
12307 {
12308 verify_merged_fold_sources(state, drop_winner, home).await?
12309 } else {
12310 Vec::new()
12311 };
12312 let root = state.worktree_root();
12313 let winner = state.tally.as_ref().map(|t| t.winner);
12314 let mut removed = Vec::new();
12315
12316 for i in 0..state.candidates.len() {
12317 let c = state.candidates[i].clone();
12318 let is_winner = Some(c.label) == winner;
12319 if is_winner && !drop_winner {
12320 continue;
12321 }
12322 let existed = c.worktree.exists();
12323 remove_preserved_worktree(&repo, &c.worktree).await?;
12326 if existed {
12327 removed.push(c.worktree.to_string_lossy().into_owned());
12328 }
12329 let handed_over = state.released_branches.contains(&c.branch);
12332 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12333 if !git::branch_delete(&repo, &c.branch).await? {
12334 bail!("candidate branch removal refused");
12335 }
12336 removed.push(c.branch.clone());
12337 }
12338 state.candidates[i].folded = true;
12339 }
12340
12341 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12342 let path = name.path();
12343 let keep = !drop_winner
12344 && winner.is_some_and(|w| {
12345 path.file_name()
12346 .is_some_and(|n| n == format!("cand-{w}").as_str())
12347 });
12348 if keep {
12349 continue;
12350 }
12351 remove_preserved_worktree(&repo, &path).await?;
12352 removed.push(path.to_string_lossy().into_owned());
12353 }
12354
12355 remove_if_empty(&root);
12364
12365 if state.enabled_worktree_config && drop_winner {
12366 git::release_worktree_config(&repo).await.ok();
12370 state.enabled_worktree_config = false;
12371 }
12372 state.save_under(home)?;
12373 Ok(removed)
12374}
12375
12376async fn verify_merged_fold_sources(
12377 state: &RunState,
12378 drop_winner: bool,
12379 home: &Path,
12380) -> Result<Vec<std::fs::File>> {
12381 let winner = state.tally.as_ref().map(|t| t.winner);
12382 let mut heads = BTreeSet::new();
12383 for candidate in &state.candidates {
12384 if Some(candidate.label) == winner && !drop_winner {
12385 continue;
12386 }
12387 anyhow::ensure!(
12388 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12389 "candidate recovery source is outside the inventoried run root: {}",
12390 candidate.worktree.display()
12391 );
12392 match std::fs::symlink_metadata(&candidate.worktree) {
12393 Ok(_) => {}
12394 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12395 Err(error) => return Err(error.into()),
12396 }
12397 if !state.released_branches.contains(&candidate.branch)
12398 && git::branch_exists(&state.repo, &candidate.branch).await?
12399 {
12400 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12401 }
12402 }
12403 let mut cargo_guards = Vec::new();
12404 let entries = match std::fs::read_dir(state.worktree_root()) {
12405 Ok(entries) => Some(entries),
12406 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12407 Err(error) => return Err(error.into()),
12408 };
12409 for entry in entries.into_iter().flatten() {
12410 let path = entry?.path();
12411 let metadata = std::fs::symlink_metadata(&path)?;
12412 anyhow::ensure!(
12413 metadata.is_dir() && !metadata.file_type().is_symlink(),
12414 "non-directory or symlink recovery source retained: {}",
12415 path.display()
12416 );
12417 if !drop_winner
12418 && winner.is_some_and(|w| {
12419 path.file_name()
12420 .is_some_and(|name| name == format!("cand-{w}").as_str())
12421 })
12422 {
12423 continue;
12424 }
12425 anyhow::ensure!(
12426 path.join(".git").is_file(),
12427 "unregistered recovery source retained: {}",
12428 path.display()
12429 );
12430 anyhow::ensure!(
12431 git::git(
12432 &path,
12433 &[
12434 "status",
12435 "--porcelain",
12436 "--untracked-files=all",
12437 "--ignored",
12438 "--",
12439 ".",
12440 ":(exclude)target"
12441 ]
12442 )
12443 .await?
12444 .is_empty(),
12445 "source changes or ignored recovery files retained: {}",
12446 path.display()
12447 );
12448 match std::fs::symlink_metadata(path.join("target")) {
12449 Ok(_) => {
12450 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12451 }
12452 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12453 Err(error) => return Err(error.into()),
12454 }
12455 for checkpoint in [
12456 "rebase-merge",
12457 "rebase-apply",
12458 "sequencer",
12459 "CHERRY_PICK_HEAD",
12460 "MERGE_HEAD",
12461 "REBASE_HEAD",
12462 "REVERT_HEAD",
12463 "BISECT_LOG",
12464 "AUTO_MERGE",
12465 "MERGE_MSG",
12466 ] {
12467 let location = git::git(
12468 &path,
12469 &[
12470 "rev-parse",
12471 "--path-format=absolute",
12472 "--git-path",
12473 checkpoint,
12474 ],
12475 )
12476 .await?;
12477 anyhow::ensure!(
12478 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12479 if e.kind() == std::io::ErrorKind::NotFound),
12480 "Git recovery checkpoint retained: {checkpoint}"
12481 );
12482 }
12483 heads.insert(git::rev_parse(&path, "HEAD").await?);
12484 }
12485 crate::fold_evidence::preserve(state, home, &heads).await?;
12486
12487 Ok(cargo_guards)
12488}
12489
12490fn remove_if_empty(dir: &Path) {
12501 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12502 std::fs::remove_dir(dir).ok();
12503 }
12504}
12505
12506pub fn worst_open(state: &RunState) -> Option<Severity> {
12508 state
12509 .reviews
12510 .last()?
12511 .reviews
12512 .iter()
12513 .flat_map(|r| r.findings.iter())
12514 .map(|f| f.severity)
12515 .max()
12516}
12517
12518async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12522 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12523 git::acquire_worktree_config(repo).await?;
12524 state.enabled_worktree_config = true;
12525 }
12526 Ok(())
12527}
12528
12529#[cfg(test)]
12530mod tests {
12531 #[tokio::test]
12532 async fn launcher_base_fetch_requires_remote_success() {
12533 let tmp = tempfile::tempdir().unwrap();
12534 let repo = tmp.path().join("repo");
12535 let remote = tmp.path().join("remote");
12536 std::fs::create_dir(&repo).unwrap();
12537 std::fs::create_dir(&remote).unwrap();
12538 git::git(&remote, &["init", "--bare", "-b", "main"])
12539 .await
12540 .unwrap();
12541 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12542 git::git(
12543 &repo,
12544 &[
12545 "-c",
12546 "user.name=Fixture",
12547 "-c",
12548 "user.email=fixture@example.invalid",
12549 "commit",
12550 "--allow-empty",
12551 "-m",
12552 "fixture",
12553 ],
12554 )
12555 .await
12556 .unwrap();
12557 git::git(
12558 &repo,
12559 &["remote", "add", "origin", remote.to_str().unwrap()],
12560 )
12561 .await
12562 .unwrap();
12563 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12564 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12565 assert_eq!(
12566 resolve_base(&repo, "main", "origin", None)
12567 .await
12568 .unwrap()
12569 .commit,
12570 head
12571 );
12572 #[cfg(unix)]
12573 {
12574 let policy: crate::config::Coordination = toml::from_str(
12575 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12576 ).unwrap();
12577 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12578 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12579 let held = crate::repo_lock::SeatLease::operation_with(
12580 &repo,
12581 Some(&policy),
12582 &crate::repo_lock::WaitPolicy::default(),
12583 )
12584 .await
12585 .unwrap();
12586 assert!(held.is_some());
12587 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12588 tokio::pin!(fetch);
12589 assert!(
12590 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12591 .await
12592 .is_err()
12593 );
12594 assert!(
12595 !repo.join(".git/FETCH_HEAD").exists(),
12596 "no fetch before admission"
12597 );
12598 drop(held);
12599 let fetched = fetch.await.unwrap();
12600 assert_eq!(fetched.commit, head);
12601 assert_eq!(fetched.remote, "origin");
12602 assert_eq!(fetched.branch, "main");
12603 }
12604 git::git(
12606 &repo,
12607 &[
12608 "remote",
12609 "set-url",
12610 "origin",
12611 tmp.path().join("missing").to_str().unwrap(),
12612 ],
12613 )
12614 .await
12615 .unwrap();
12616 let error = resolve_base(&repo, "main", "origin", None)
12617 .await
12618 .unwrap_err()
12619 .to_string();
12620 assert!(error.contains("cannot read origin/main"));
12621 assert!(error.contains("refusing to branch off"));
12622 }
12623
12624 #[test]
12625 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12626 let leaked = vec!["chore(deps): update a crate".to_owned()];
12627 let own = "fix(daemon): apply a chosen action";
12628 assert!(should_retitle("", own, &leaked));
12629 assert!(should_retitle(
12630 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12631 own,
12632 &leaked
12633 ));
12634 assert!(should_retitle(
12635 "chore: land candidate A of run 1",
12636 own,
12637 &leaked
12638 ));
12639 assert!(should_retitle(
12640 "magi: candidate A (uncommitted work)",
12641 own,
12642 &leaked
12643 ));
12644 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12645 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12646 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12647 }
12648
12649 #[test]
12650 fn pr_merge_plan_creates_adopts_or_stops() {
12651 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12652 assert_eq!(
12653 pr_merge_plan(Ok(land::OpenPr::One {
12654 url: "u".into(),
12655 title: "t".into()
12656 })),
12657 PrPlan::Adopt {
12658 url: "u".into(),
12659 title: "t".into()
12660 }
12661 );
12662 let PrPlan::Stop(many) =
12663 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12664 else {
12665 panic!("many must stop");
12666 };
12667 assert!(many.contains('a') && many.contains('b'));
12668 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12669 panic!("a failed lookup must stop");
12670 };
12671 assert!(err.contains("bad token"));
12672 }
12673
12674 use super::*;
12675 use crate::run::GateStatus;
12676 use std::collections::BTreeMap;
12677 use std::time::Duration;
12678
12679 fn conductor() -> AgentSpec {
12680 AgentSpec {
12681 id: "conductor".to_owned(),
12682 kind: crate::config::AgentKind::Command,
12683 model: None,
12684 command: vec!["true".to_owned()],
12685 extra_args: Vec::new(),
12686 env: BTreeMap::new(),
12687 prompt_delivery: None,
12688 max_turn_secs: None,
12689 }
12690 }
12691
12692 fn spec(id: &str) -> AgentSpec {
12693 AgentSpec {
12694 id: id.to_owned(),
12695 kind: crate::config::AgentKind::Command,
12696 model: None,
12697 command: vec!["true".to_owned()],
12698 extra_args: Vec::new(),
12699 env: BTreeMap::new(),
12700 prompt_delivery: None,
12701 max_turn_secs: None,
12702 }
12703 }
12704
12705 fn ids(xs: &[&str]) -> BTreeSet<String> {
12706 xs.iter().map(|s| (*s).to_owned()).collect()
12707 }
12708
12709 #[test]
12710 fn pick_successor_skips_an_agent_another_seat_holds() {
12711 let roster = [spec("a"), spec("b"), spec("c")];
12713 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12714 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12715 }
12716
12717 #[test]
12718 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12719 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12721 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12722 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12723 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12724 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12725 }
12726
12727 #[test]
12728 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12729 let roster = [spec("a"), spec("b")];
12730 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12731 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12732 let carried = ids(&["b"]);
12733 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12734 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12735 }
12736
12737 #[test]
12738 fn pick_successor_returns_none_without_any_untried_successor() {
12739 let roster = [spec("a"), spec("b")];
12740 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12741 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12742 let carried = ids(&["a"]);
12743 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12744 }
12745
12746 #[test]
12747 fn pick_successor_rescue_avoids_another_seats_occupant() {
12748 let roster = [spec("a"), spec("b"), spec("c")];
12750 let carried = ids(&["b", "c"]);
12751 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12752 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12753 }
12754
12755 #[test]
12756 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12757 let roster = [spec("a"), spec("b"), spec("c")];
12758 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12759 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12760 }
12761
12762 #[test]
12763 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12764 let roster = [spec("a"), spec("b"), spec("c")];
12765 let failed = ids(&["a", "b", "c"]);
12766 let mut tried = ids(&["b"]);
12768 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12769 assert_eq!(first.id, "a");
12770 tried.insert(first.id.clone());
12771 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12772 assert_eq!(second.id, "c");
12773 tried.insert(second.id.clone());
12774 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12775 }
12776
12777 #[test]
12778 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12779 let roster = [spec("a"), spec("b")];
12780 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12781 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12782 }
12783
12784 #[test]
12785 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12786 let roster = [spec("a"), spec("b"), spec("c")];
12787 let h = SeatHistory {
12788 failed: ids(&["a"]),
12789 last_ok: Some("b".to_owned()),
12790 last_fail: None,
12791 };
12792 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12793 let gone = SeatHistory {
12795 last_ok: Some("zzz".to_owned()),
12796 ..h.clone()
12797 };
12798 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12799 let failed = SeatHistory {
12800 failed: ids(&["a", "b"]),
12801 last_ok: Some("b".to_owned()),
12802 last_fail: None,
12803 };
12804 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12805 }
12806
12807 #[test]
12808 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12809 let first = SeatState::new("review-1", "alpha", 7);
12810 let next = handover_seat("review-1", "gamma", 7);
12811 assert_ne!(first.claude_session, next.claude_session);
12812 }
12813
12814 #[test]
12815 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12816 let mut state = state_with_summary("x", "y");
12817 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12818 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12819 assert_ne!(a.claude_session, b.claude_session);
12820 }
12821
12822 #[test]
12823 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12824 let roster = [spec("a"), spec("b")];
12825 let h = SeatHistory {
12826 failed: ids(&["a", "b"]),
12827 ..SeatHistory::default()
12828 };
12829 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12830 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12831 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12832 }
12833
12834 #[test]
12841 fn failure_signature_ignores_numbers_and_paths() {
12842 assert_eq!(
12843 failure_signature("exited with Some(2) and no usable output"),
12844 failure_signature("exited with Some(137) and no usable output")
12845 );
12846 assert_eq!(
12847 failure_signature("process exited with no usable output (exit code 2)"),
12848 failure_signature("process exited with no usable output (exit code 137)")
12849 );
12850 for (old, new) in [
12851 (
12852 "exited with Some(2) and no usable output",
12853 "process exited with no usable output (exit code 2)",
12854 ),
12855 (
12856 "exited with None and no usable output",
12857 "process exited with no usable output (exit code unavailable)",
12858 ),
12859 ] {
12860 let previous = FailClass::Other(failure_signature(old));
12861 let current = FailClass::Other(failure_signature(new));
12862 assert_eq!(previous, current, "resumption retains the failure class");
12863 assert!(!should_hand_over(Some(&previous), ¤t));
12864 }
12865 assert_eq!(
12866 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12867 failure_signature("cannot open /var/x.txt: denied")
12868 );
12869 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12870 }
12871
12872 #[test]
12873 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12874 let other = FailClass::Other("x".into());
12875 assert!(should_hand_over(None, &FailClass::Quota));
12876 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12877 assert!(should_hand_over(
12878 Some(&FailClass::Timeout),
12879 &FailClass::Timeout
12880 ));
12881 assert!(should_hand_over(None, &other));
12882 assert!(!should_hand_over(Some(&other), &other));
12883 assert!(should_hand_over(
12884 Some(&other),
12885 &FailClass::Other("y".into())
12886 ));
12887 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12889 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12890 }
12891
12892 #[test]
12893 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12894 let a = SeatState::new("judge-1", "alpha", 7);
12895 let b = handover_seat("judge-1", "beta", 7);
12896 assert_ne!(a.claude_session, b.claude_session);
12897 assert_eq!(b.turns, 0);
12898 }
12899
12900 #[test]
12901 fn a_timeout_is_classified_apart_from_other_failures() {
12902 assert_eq!(
12903 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12904 Some(FailClass::Timeout)
12905 );
12906 assert!(matches!(
12907 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12908 Some(FailClass::Other(_))
12909 ));
12910 }
12911
12912 #[test]
12913 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12914 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12915 let tried = BTreeSet::from(["beta".to_owned()]);
12916 let next = next_untried_in_roster(&roster, 1, &tried);
12919 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12920 }
12921
12922 #[test]
12923 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12924 let roster = vec![spec("alpha"), spec("beta")];
12925 let tried = BTreeSet::from(["beta".to_owned()]);
12926 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12930 }
12931
12932 #[test]
12933 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12934 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12935 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12936 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12940 }
12941
12942 #[test]
12943 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12944 let roster = vec![spec("a"), spec("a"), spec("b")];
12945 let tried = BTreeSet::from(["a".to_owned()]);
12946 let next = next_untried_in_roster(&roster, 0, &tried);
12947 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12948 }
12949
12950 #[test]
12951 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12952 let roster = vec![spec("a"), spec("b")];
12953 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12954 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12955 }
12956
12957 #[test]
12958 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12959 let dir = tempfile::tempdir().unwrap();
12960 let bay = dir.path().join("ffff");
12961
12962 remove_if_empty(&bay);
12964 assert!(!bay.exists());
12965
12966 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12969 remove_if_empty(&bay);
12970 assert!(bay.exists(), "non-empty directory must survive");
12971
12972 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12974 remove_if_empty(&bay);
12975 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12976 }
12977
12978 #[test]
12987 fn a_full_panel_that_found_nothing_is_clean() {
12988 assert!(round_is_clean(
12989 0,
12990 true,
12991 2,
12992 2,
12993 0,
12994 IncompleteReviewPolicy::Block
12995 ));
12996 }
12997
12998 #[test]
12999 fn a_missing_seat_is_never_clean_under_the_default_policy() {
13000 assert!(!round_is_clean(
13001 0,
13002 true,
13003 1,
13004 2,
13005 0,
13006 IncompleteReviewPolicy::Block
13007 ));
13008 }
13009
13010 #[test]
13011 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13012 assert!(!round_is_clean(
13013 1,
13014 true,
13015 1,
13016 2,
13017 0,
13018 IncompleteReviewPolicy::Warn
13019 ));
13020 }
13021
13022 #[test]
13023 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13024 assert!(round_is_clean(
13025 0,
13026 true,
13027 1,
13028 2,
13029 0,
13030 IncompleteReviewPolicy::Warn
13031 ));
13032 }
13033
13034 #[test]
13035 fn a_full_panel_with_an_open_finding_is_not_clean() {
13036 assert!(!round_is_clean(
13037 1,
13038 true,
13039 2,
13040 2,
13041 0,
13042 IncompleteReviewPolicy::Block
13043 ));
13044 }
13045
13046 #[test]
13047 fn a_full_panel_with_a_red_e2e_is_not_clean() {
13048 assert!(!round_is_clean(
13049 0,
13050 false,
13051 2,
13052 2,
13053 0,
13054 IncompleteReviewPolicy::Block
13055 ));
13056 }
13057
13058 #[test]
13065 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13066 assert!(round_is_clean(
13069 0,
13070 true,
13071 1,
13072 2,
13073 1,
13074 IncompleteReviewPolicy::Block
13075 ));
13076 }
13077
13078 #[test]
13079 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13080 assert!(!round_is_clean(
13083 0,
13084 true,
13085 1,
13086 2,
13087 0,
13088 IncompleteReviewPolicy::Block
13089 ));
13090 }
13091
13092 #[test]
13093 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13094 assert!(!round_is_clean(
13095 1,
13096 true,
13097 1,
13098 2,
13099 1,
13100 IncompleteReviewPolicy::Block
13101 ));
13102 assert!(!round_is_clean(
13103 0,
13104 false,
13105 1,
13106 2,
13107 1,
13108 IncompleteReviewPolicy::Block
13109 ));
13110 }
13111
13112 #[test]
13113 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
13114 assert!(!round_is_clean(
13118 0,
13119 true,
13120 0,
13121 2,
13122 2,
13123 IncompleteReviewPolicy::Block
13124 ));
13125 }
13126
13127 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
13128 CommandOutcome {
13129 command: "test".to_owned(),
13130 code,
13131 output_tail: String::new(),
13132 duration_ms: 0,
13133 resource_blocked,
13134 }
13135 }
13136
13137 #[test]
13138 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
13139 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
13140 assert!(
13141 !verify_inconclusive(&[outcome(Some(1), false)]),
13142 "an ordinary failure is still evidence about the patch"
13143 );
13144 assert!(verify_inconclusive(&[outcome(None, true)]));
13145 assert!(
13146 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
13147 "one inconclusive outcome taints the whole batch"
13148 );
13149 assert!(!verify_inconclusive(&[]));
13150 }
13151
13152 #[tokio::test]
13153 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
13154 let calls = std::sync::atomic::AtomicUsize::new(0);
13158 let started = Instant::now();
13159 wait_for_pids_with(
13160 &[123],
13161 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
13162 Duration::from_millis(5),
13163 Duration::from_secs(5),
13164 )
13165 .await;
13166 assert!(
13167 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
13168 "must keep checking rather than deciding on the first answer"
13169 );
13170 assert!(
13171 started.elapsed() < Duration::from_secs(1),
13172 "must return the moment it is confirmed dead, not wait out the ceiling"
13173 );
13174 }
13175
13176 #[tokio::test]
13177 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
13178 let started = Instant::now();
13179 wait_for_pids_with(
13180 &[123],
13181 |_| true, Duration::from_millis(5),
13183 Duration::from_millis(30),
13184 )
13185 .await;
13186 let elapsed = started.elapsed();
13187 assert!(
13188 elapsed >= Duration::from_millis(30),
13189 "must not give up before its own ceiling: {elapsed:?}"
13190 );
13191 assert!(
13192 elapsed < Duration::from_secs(1),
13193 "must not wait past its own ceiling either: {elapsed:?}"
13194 );
13195 }
13196
13197 #[tokio::test]
13198 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
13199 let started = Instant::now();
13200 wait_for_pids_with(
13201 &[],
13202 |_| true,
13203 Duration::from_secs(5),
13204 Duration::from_secs(5),
13205 )
13206 .await;
13207 assert!(
13208 started.elapsed() < Duration::from_millis(200),
13209 "an empty pid list has nothing to confirm"
13210 );
13211 }
13212
13213 fn review_round(
13219 clean: bool,
13220 blocking: usize,
13221 answered: usize,
13222 expected: usize,
13223 progressed: bool,
13224 e2e_ok: bool,
13225 ) -> ReviewRound {
13226 ReviewRound {
13227 round: 1,
13228 head: "h".to_owned(),
13229 verified_head: None,
13230 verified_at: None,
13231 reviews: Vec::new(),
13232 e2e: vec![CommandOutcome {
13233 command: "test".to_owned(),
13234 code: Some(if e2e_ok { 0 } else { 1 }),
13235 output_tail: String::new(),
13236 duration_ms: 0,
13237 resource_blocked: false,
13238 }],
13239 verify_retried: false,
13240 e2e_deferred: false,
13241 e2e_defer_reason: None,
13242 fix: None,
13243 blocking,
13244 answered,
13245 expected,
13246 clean,
13247 progressed,
13248 vote_split: false,
13249 reconsideration: Vec::new(),
13250 verdict: None,
13251 }
13252 }
13253
13254 #[test]
13255 fn review_conclusion_is_none_when_nothing_has_run() {
13256 assert_eq!(review_conclusion(&[], 3), None);
13257 }
13258
13259 #[test]
13260 fn review_conclusion_is_none_while_rounds_remain() {
13261 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
13262 assert_eq!(review_conclusion(&rounds, 3), None);
13263 }
13264
13265 #[test]
13266 fn review_conclusion_is_gating_once_a_round_is_clean() {
13267 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
13268 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
13269 }
13270
13271 #[test]
13272 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
13273 let rounds = vec![
13274 review_round(false, 1, 2, 2, true, true),
13275 review_round(false, 1, 2, 2, true, true),
13276 ];
13277 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
13278 }
13279
13280 #[test]
13281 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
13282 let rounds = vec![
13283 review_round(false, 1, 2, 2, true, true),
13284 review_round(false, 1, 2, 2, true, false),
13285 ];
13286 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
13287 }
13288
13289 #[test]
13290 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
13291 let mut blocked = review_round(false, 1, 2, 2, true, false);
13298 blocked.e2e[0].resource_blocked = true;
13299 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
13300 assert_eq!(review_conclusion(&rounds, 2), None);
13301 }
13302
13303 #[test]
13304 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
13305 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
13307 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
13308 }
13309
13310 #[test]
13311 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
13312 let rounds = vec![
13313 review_round(false, 1, 2, 2, false, true),
13314 review_round(false, 1, 2, 2, false, true),
13315 ];
13316 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
13317 }
13318
13319 fn secs(n: u64) -> Duration {
13320 Duration::from_secs(n)
13321 }
13322
13323 #[tokio::test]
13326 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13327 const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
13328 if std::env::var_os(CHILD).is_none() {
13329 let output = std::process::Command::new(std::env::current_exe().unwrap())
13333 .args([
13334 "--exact",
13335 "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
13336 "--nocapture",
13337 ])
13338 .env(CHILD, "1")
13339 .env_remove("MAGI_WORKER_HOME")
13340 .env_remove("MAGI_WORKER_ID")
13341 .output()
13342 .unwrap();
13343 assert!(
13344 output.status.success()
13345 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
13346 "{}\n{}",
13347 String::from_utf8_lossy(&output.stdout),
13348 String::from_utf8_lossy(&output.stderr)
13349 );
13350 return;
13351 }
13352 ask_test_home();
13353 let tmp = tempfile::tempdir().unwrap();
13354 let repo = tmp.path().join("repo");
13355 std::fs::create_dir_all(&repo).unwrap();
13356 init_repo(&repo);
13357 let g = |args: &[&str]| {
13358 let out = std::process::Command::new("git")
13359 .args(args)
13360 .current_dir(&repo)
13361 .quiet()
13362 .output()
13363 .unwrap();
13364 assert!(
13365 out.status.success(),
13366 "{}",
13367 String::from_utf8_lossy(&out.stderr)
13368 );
13369 };
13370 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13371 g(&["checkout", "-b", "historical"]);
13372 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13373 g(&["add", "."]);
13374 g(&["commit", "-m", "old alternative"]);
13375 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13376 g(&["checkout", "-b", "magi/task-selected", &base]);
13377 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13378 g(&["add", "."]);
13379 g(&["commit", "-m", "measurement infrastructure"]);
13380 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13381 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13382 g(&["add", "."]);
13383 g(&["commit", "-m", "dependent implementation"]);
13384 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13385 g(&["checkout", "main"]);
13386 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13387 g(&["add", "."]);
13388 g(&["commit", "-m", "unrelated base change"]);
13389 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13390 let origin = tmp.path().join("origin.git");
13391 g(&[
13392 "clone",
13393 "--bare",
13394 repo.to_str().unwrap(),
13395 origin.to_str().unwrap(),
13396 ]);
13397 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13398 g(&["fetch", "origin"]);
13399 let selection = crate::pr_recovery::Selection {
13400 url: "https://example.test/project/pull/1".into(),
13401 branch: "magi/task-selected".into(),
13402 head: head.clone(),
13403 source_head: None,
13404 source_run: None,
13405 source_proof: None,
13406 disposition: Some(crate::pr_recovery::Disposition::Review),
13407 instruction: "preserve measured behavior".into(),
13408 approved_at: jiff::Timestamp::now(),
13409 };
13410 let mut template = runner_at(RunStatus::Reviewing);
13411 template.state.repo = repo.clone();
13412 template.state.base_commit = current.clone();
13413 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13414 template.state.origin = Some(crate::run::Origin::queue("selected"));
13415 template.state.pr_head = Some(selection.branch.clone());
13416 template.state.pr_recovery = Some(selection.clone());
13417 let mut runner =
13418 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13419 .await
13420 .unwrap();
13421 let wt = runner.state.winner().unwrap().worktree.clone();
13422 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13423 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13424 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13425 assert!(runner.state.seeds.is_empty());
13426 assert_eq!(runner.state.instruction, "preserve measured behavior");
13427 runner.sync_to_base().await.unwrap();
13428 assert_eq!(
13429 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13430 "prerequisite"
13431 );
13432 assert_eq!(
13433 std::fs::read_to_string(wt.join("implementation")).unwrap(),
13434 "measured implementation"
13435 );
13436 assert_eq!(
13437 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13438 "current main change"
13439 );
13440 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13441 assert_eq!(
13442 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13443 head
13444 );
13445
13446 let mut partial = runner_at(RunStatus::Reviewing);
13448 partial.state.repo = repo.clone();
13449 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13450 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13451 run: Some("exit 1".into()),
13452 ..Default::default()
13453 }];
13454 let selection = runner.state.pr_recovery.clone().unwrap();
13455 partial.state.pr_recovery = Some(selection.clone());
13456 partial.state.pr_head = Some(selection.branch.clone());
13457 partial.state.origin = Some(crate::run::Origin::queue("selected"));
13458 let id = partial.state.id.clone();
13459 let error = match Runner::open_selected(
13460 &repo,
13461 partial.state,
13462 partial.roles,
13463 1,
13464 runner.state.base_commit.clone(),
13465 selection,
13466 )
13467 .await
13468 {
13469 Ok(_) => panic!("setup must fail"),
13470 Err(e) => e,
13471 };
13472 assert_eq!(
13473 error
13474 .downcast_ref::<crate::pr_recovery::StartupFailed>()
13475 .unwrap()
13476 .run,
13477 id
13478 );
13479 let failed = RunState::load(&id).unwrap();
13480 assert_eq!(failed.status, RunStatus::Blocked);
13481 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13482 assert!(Runner::resume(&id).is_err());
13483 runner.state.status = RunStatus::Ready;
13484 runner.state.driver_exited = true;
13485 runner.state.save().unwrap();
13486 let mut retry = runner_at(RunStatus::Reviewing);
13487 retry.state.repo = repo.clone();
13488 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13489 let mut reselected = runner.state.pr_recovery.clone().unwrap();
13490 reselected.approved_at = jiff::Timestamp::now();
13491 retry.state.origin = Some(crate::run::Origin::queue("selected"));
13492 retry.state.pr_head = Some(reselected.branch.clone());
13493 retry.state.pr_recovery = Some(reselected.clone());
13494 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13495 .await
13496 .unwrap();
13497 let restarted = Runner::open_selected(
13498 &repo,
13499 retry.state,
13500 retry.roles,
13501 1,
13502 runner.state.base_commit.clone(),
13503 reselected,
13504 )
13505 .await
13506 .unwrap();
13507 assert_eq!(
13508 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13509 .await
13510 .unwrap(),
13511 head
13512 );
13513 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13514 assert_eq!(
13515 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13516 head
13517 );
13518 }
13519
13520 #[tokio::test]
13521 #[cfg(unix)]
13522 async fn fallback_and_interrupted_restoration_require_coordination() {
13523 use std::os::fd::AsRawFd as _;
13524 ask_test_home();
13525 let tmp = tempfile::tempdir().unwrap();
13526 let mut runner = registered_runner(tmp.path()).await;
13527 runner.state.config.graph.timeout_resource_wait = 1;
13528 let winner = runner.state.candidates[0].clone();
13529 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13530 let leftover = winner.worktree.join("leftover");
13531 std::fs::write(&leftover, "untrusted edits").unwrap();
13532 let next = tmp.path().join("next-fixer");
13533 let mut failed = spec("first");
13534 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13535 let mut successor = spec("second");
13536 successor.command = vec![
13537 "sh".into(),
13538 "-c".into(),
13539 format!("touch '{}'; echo done", next.display()),
13540 ];
13541 let attempts = vec![
13542 (failed.clone(), "first".into()),
13543 (successor.clone(), "second".into()),
13544 ];
13545 let lock_path = runner.state.repo.join(".git/updater.lock");
13546 let external = std::fs::File::create(&lock_path).unwrap();
13547 assert_eq!(
13548 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13549 0
13550 );
13551 let artifacts = tmp.path().join("artifacts");
13552 let build = |spec, seat| SeatJob {
13553 spec,
13554 seat,
13555 cwd: winner.worktree.clone(),
13556 prompt: "fixture".into(),
13557 timeout: Duration::from_secs(10),
13558 allow_write: true,
13559 sessions: false,
13560 artifacts: artifacts.clone(),
13561 stem: "fix".into(),
13562 handover: None,
13563 };
13564 let (_, _, out) = runner
13565 .ask_fixer_among(
13566 attempts.clone(),
13567 "graft",
13568 None,
13569 Some((&winner.worktree, &head)),
13570 build,
13571 )
13572 .await;
13573 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13574 assert!(leftover.exists(), "contention must prevent clean");
13575 assert!(!next.exists());
13576 assert!(runner.state.handovers.is_empty());
13577 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13579 from_run: "earlier".into(),
13580 generation: 1,
13581 mode: crate::recovery::Mode::VerifyRepair,
13582 reasons: Vec::new(),
13583 reviewed_head: None,
13584 rounds: Vec::new(),
13585 open_findings: Vec::new(),
13586 evidence: None,
13587 criteria_inherited: true,
13588 repair: crate::recovery::Repair::Started {
13589 before: head.clone(),
13590 },
13591 });
13592 assert!(
13593 runner
13594 .recovery_repair(&winner)
13595 .await
13596 .unwrap_err()
13597 .is::<crate::repo_lock::ResourceBlocked>()
13598 );
13599 assert!(matches!(
13600 runner.state.recovery.as_ref().unwrap().repair,
13601 crate::recovery::Repair::Started { .. }
13602 ));
13603 assert!(leftover.exists());
13604 drop(external);
13605 runner.recovery_repair(&winner).await.unwrap();
13606 assert!(!leftover.exists());
13607 assert!(matches!(
13608 runner.state.recovery.as_ref().unwrap().repair,
13609 crate::recovery::Repair::Done { .. }
13610 ));
13611 std::fs::remove_file(&lock_path).unwrap();
13613 std::fs::create_dir(&lock_path).unwrap();
13614 std::fs::write(&leftover, "untrusted edits").unwrap();
13615 let (_, _, out) = runner
13616 .ask_fixer_among(
13617 attempts,
13618 "graft",
13619 None,
13620 Some((&winner.worktree, &head)),
13621 build,
13622 )
13623 .await;
13624 assert!(matches!(out, AgentOutcome::Failed(_)));
13625 assert!(leftover.exists());
13626 assert!(!next.exists());
13627 assert_eq!(
13628 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13629 head
13630 );
13631 }
13632
13633 #[cfg(unix)]
13634 async fn registered_runner(root: &Path) -> Runner {
13635 ask_test_home();
13636 let repo = root.join("repo");
13637 std::fs::create_dir_all(&repo).unwrap();
13638 init_repo(&repo);
13639 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13640 let mut runner = runner_at(RunStatus::Prep);
13641 runner.state.repo = repo.clone();
13642 runner.state.config.coordination = crate::config::test_coordination();
13643 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13644 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13645 runner.state.config.graph.advise = false;
13646 runner.roles.implementers = vec![spec("implement")];
13647 runner.roles.judges = vec![spec("judge")];
13648 runner.prep().await.unwrap();
13649 runner
13650 }
13651
13652 #[tokio::test]
13653 #[cfg(unix)]
13654 async fn parked_implementer_persists_completed_work() {
13655 for registered in [false, true] {
13656 let tmp = tempfile::tempdir().unwrap();
13657 let mut runner = registered_runner(tmp.path()).await;
13658 if !registered {
13659 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13660 }
13661 let marker = tmp.path().join("started");
13662 let release = tmp.path().join("release");
13663 let mut implementer = spec("implement");
13664 implementer.command = vec![
13665 "sh".into(),
13666 "-c".into(),
13667 format!(
13668 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13669 marker.display(),
13670 release.display()
13671 ),
13672 ];
13673 runner.roles.implementers = vec![implementer.clone()];
13674 runner.state.config.agents = vec![implementer];
13675 let pause = runner.pause.clone();
13676 let (result, ()) = tokio::join!(runner.implement(), async {
13677 tokio::time::timeout(Duration::from_secs(10), async {
13678 while !marker.exists() {
13679 tokio::time::sleep(Duration::from_millis(10)).await;
13680 }
13681 })
13682 .await
13683 .unwrap();
13684 pause.park();
13685 std::fs::write(&release, "go").unwrap();
13686 });
13687 result.unwrap();
13688 let candidate = &runner.state.candidates[0];
13689 assert!(candidate.commits > 0, "{candidate:?}");
13690 assert!(candidate.summary.contains("saved-result"));
13691 assert_eq!(candidate.files, 1);
13692 let before = candidate.commits;
13693 runner.implement().await.unwrap();
13694 assert_eq!(runner.state.candidates[0].commits, before);
13695 }
13696 }
13697
13698 #[tokio::test]
13699 #[cfg(unix)]
13700 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13701 let tmp = tempfile::tempdir().unwrap();
13702 let mut runner = registered_runner(tmp.path()).await;
13703 let winner = runner.state.candidates[0].clone();
13704 let cache = tmp.path().join("cache");
13705 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13706 runner.state.config.graph.timeout_resource_wait = 0;
13707 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13708 let crate::cache::AcquireOutcome::Acquired(held) =
13709 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13710 else {
13711 panic!("cache must be free")
13712 };
13713 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13714 from_run: "earlier".into(),
13715 generation: 1,
13716 mode: crate::recovery::Mode::VerifyRepair,
13717 reasons: Vec::new(),
13718 reviewed_head: None,
13719 rounds: Vec::new(),
13720 open_findings: Vec::new(),
13721 evidence: None,
13722 criteria_inherited: true,
13723 repair: crate::recovery::Repair::NotStarted,
13724 });
13725 runner.recovery_repair(&winner).await.unwrap();
13726 assert!(matches!(
13727 runner.state.recovery.as_ref().unwrap().repair,
13728 crate::recovery::Repair::WaitingForRestore { .. }
13729 ));
13730 drop(held);
13731 runner.state.resource_block = None;
13732 runner.state.config.graph.timeout_resource_wait = 1;
13733 runner.recovery_repair(&winner).await.unwrap();
13734 assert!(matches!(
13735 runner.state.recovery.as_ref().unwrap().repair,
13736 crate::recovery::Repair::Done { .. }
13737 ));
13738 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13739 }
13740
13741 #[tokio::test]
13742 #[cfg(unix)]
13743 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13744 use std::os::fd::AsRawFd as _;
13745 for graft in [false, true] {
13746 let tmp = tempfile::tempdir().unwrap();
13747 let mut runner = registered_runner(tmp.path()).await;
13748 runner.state.config.graph.timeout_resource_wait = 1;
13749 let winner = runner.state.candidates[0].clone();
13750 let marker = tmp.path().join("fixer-started");
13751 let mut fixer = spec("fixer");
13752 fixer.command = vec![
13753 "sh".into(),
13754 "-c".into(),
13755 format!("echo started >> '{}'; echo done", marker.display()),
13756 ];
13757 runner.roles.fixer = Some(vec![fixer]);
13758 if graft {
13759 let mut loser = winner.clone();
13760 loser.label = 'B';
13761 loser.index = 1;
13762 runner.state.candidates.push(loser);
13763 runner.state.config.graph.graft_runner_up = true;
13764 runner.state.tally = Some(Tally {
13765 first_choice: BTreeMap::new(),
13766 borda: BTreeMap::new(),
13767 winner: winner.label,
13768 rankings: 1,
13769 unanimous_initial: true,
13770 deliberated: false,
13771 changed_votes: 0,
13772 unanimous_final: true,
13773 tie_break: None,
13774 judges: 1,
13775 present: 1,
13776 quorum: 1,
13777 met_quorum: true,
13778 uncontested: None,
13779 });
13780 runner.state.judgements.push(crate::run::Judgement {
13781 judge: 1,
13782 seat: "judge-1".into(),
13783 agent: "judge".into(),
13784 ranking: vec![winner.label, 'B'],
13785 reasons: BTreeMap::new(),
13786 confidence: None,
13787 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13788 order: vec![0, 1],
13789 failed: None,
13790 duration_ms: 0,
13791 });
13792 } else {
13793 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13794 from_run: "earlier".into(),
13795 generation: 1,
13796 mode: crate::recovery::Mode::VerifyRepair,
13797 reasons: Vec::new(),
13798 reviewed_head: None,
13799 rounds: Vec::new(),
13800 open_findings: Vec::new(),
13801 evidence: None,
13802 criteria_inherited: true,
13803 repair: crate::recovery::Repair::NotStarted,
13804 });
13805 }
13806 let reader = crate::repo_lock::SeatLease::for_checkout(
13807 &winner.worktree,
13808 false,
13809 crate::config::test_coordination().as_ref(),
13810 )
13811 .await
13812 .unwrap();
13813 let pause = runner.pause.clone();
13814 let (result, ()) = tokio::join!(
13815 async {
13816 if graft {
13817 runner.graft().await
13818 } else {
13819 runner.recovery_repair(&winner).await
13820 }
13821 },
13822 async {
13823 tokio::time::sleep(Duration::from_millis(100)).await;
13824 pause.park();
13825 }
13826 );
13827 result.unwrap();
13828 assert!(!marker.exists(), "a blocked seat must not spawn");
13829 runner.state = RunState::load(&runner.state.id).unwrap();
13831 if graft {
13832 assert_eq!(
13833 runner.state.graft.as_ref().unwrap().outcome,
13834 GraftOutcome::WaitingForRestore
13835 );
13836 } else {
13837 assert!(matches!(
13838 runner.state.recovery.as_ref().unwrap().repair,
13839 crate::recovery::Repair::WaitingForRestore { .. }
13840 ));
13841 }
13842 drop(reader);
13843 runner.pause = Pause::new();
13844 runner.state.resource_block = None;
13845 let external =
13847 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13848 assert_eq!(
13849 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13850 0
13851 );
13852 let leftover = winner.worktree.join("leftover");
13853 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13854 let result = if graft {
13855 runner.graft().await
13856 } else {
13857 runner.recovery_repair(&winner).await
13858 };
13859 assert!(
13860 result
13861 .unwrap_err()
13862 .is::<crate::repo_lock::ResourceBlocked>()
13863 );
13864 assert!(leftover.exists());
13865 assert!(!marker.exists());
13866 drop(external);
13867 runner.state.resource_block = None;
13868 let result = if graft {
13869 runner.graft().await
13870 } else {
13871 runner.recovery_repair(&winner).await
13872 };
13873 result.unwrap();
13874 assert!(!leftover.exists(), "restore must precede retry");
13875 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13876 if graft {
13877 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13878 } else {
13879 assert!(matches!(
13880 runner.state.recovery.as_ref().unwrap().repair,
13881 crate::recovery::Repair::Done { .. }
13882 ));
13883 }
13884 }
13885 }
13886
13887 #[tokio::test]
13888 #[cfg(unix)]
13889 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13890 use std::os::fd::AsRawFd as _;
13891 let tmp = tempfile::tempdir().unwrap();
13892 let mut runner = registered_runner(tmp.path()).await;
13893 runner.state.config.graph.timeout_resource_wait = 1;
13894 let winner = runner.state.candidates[0].clone();
13895 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13896 runner.state.tally = Some(Tally {
13897 first_choice: BTreeMap::new(),
13898 borda: BTreeMap::new(),
13899 winner: winner.label,
13900 rankings: 0,
13901 unanimous_initial: false,
13902 deliberated: false,
13903 changed_votes: 0,
13904 unanimous_final: false,
13905 tie_break: None,
13906 judges: 0,
13907 present: 0,
13908 quorum: 0,
13909 met_quorum: true,
13910 uncontested: Some("fixture".into()),
13911 });
13912 runner.state.graft = Some(GraftRecord {
13913 pre: pre.clone(),
13914 post: None,
13915 lines: None,
13916 limit: None,
13917 agent: None,
13918 outcome: GraftOutcome::Running,
13919 requested: Vec::new(),
13920 adoption: None,
13921 adoption_error: None,
13922 });
13923 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13924 git::commit_all(&winner.worktree, "unfinished graft")
13925 .await
13926 .unwrap();
13927 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13928 let leftover = winner.worktree.join("leftover");
13929 std::fs::write(&leftover, "untrusted edits").unwrap();
13930 let path = runner.state.repo.join(".git/updater.lock");
13931 let external = std::fs::File::create(&path).unwrap();
13932 assert_eq!(
13933 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13934 0
13935 );
13936 assert!(
13937 runner
13938 .graft()
13939 .await
13940 .unwrap_err()
13941 .is::<crate::repo_lock::ResourceBlocked>()
13942 );
13943 assert!(matches!(
13944 runner.state.graft.as_ref().unwrap().outcome,
13945 GraftOutcome::Running
13946 ));
13947 assert_eq!(
13948 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13949 unfinished
13950 );
13951 assert!(leftover.exists());
13952 drop(external);
13953 std::fs::remove_file(&path).unwrap();
13954 std::fs::create_dir(&path).unwrap();
13955 assert!(runner.graft().await.is_err());
13956 assert!(matches!(
13957 runner.state.graft.as_ref().unwrap().outcome,
13958 GraftOutcome::Running
13959 ));
13960 assert_eq!(
13961 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13962 unfinished
13963 );
13964 assert!(leftover.exists());
13965 std::fs::remove_dir(&path).unwrap();
13966 runner.graft().await.unwrap();
13967 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13968 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13969 assert!(!leftover.exists());
13970 }
13971
13972 #[tokio::test]
13973 #[cfg(unix)]
13974 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13975 use std::os::fd::AsRawFd as _;
13976 let tmp = tempfile::tempdir().unwrap();
13977 let mut runner = registered_runner(tmp.path()).await;
13978 runner.state.prep_complete = Some(false);
13979 runner.state.config.graph.timeout_resource_wait = 1;
13980 runner.state.save().unwrap();
13981 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13982 assert_eq!(
13983 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13984 0
13985 );
13986 runner.execute().await.unwrap();
13987 let disk = RunState::load(&runner.state.id).unwrap();
13988 assert!(disk.driver_exited);
13989 assert_eq!(disk.driver_pid, None);
13990 assert_eq!(runner.state.status, RunStatus::Blocked);
13991 assert_eq!(
13992 runner.state.resource_block.as_ref().unwrap().kind,
13993 "repository coordination"
13994 );
13995 assert!(runner.state.prep_failure.is_none());
13996 assert!(runner.state.seats.is_empty());
13997 runner.pause.park();
13998 tokio::time::timeout(Duration::from_secs(10), runner.execute())
14001 .await
14002 .unwrap()
14003 .unwrap();
14004 assert!(runner.state.parked);
14005 assert!(runner.state.prep_failure.is_none());
14006 let disk = RunState::load(&runner.state.id).unwrap();
14007 assert!(disk.driver_exited);
14008 assert_eq!(disk.driver_pid, None);
14009 }
14010
14011 #[tokio::test]
14012 #[cfg(unix)]
14013 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
14014 use std::os::fd::AsRawFd as _;
14015 let tmp = tempfile::tempdir().unwrap();
14016 let mut runner = registered_runner(tmp.path()).await;
14017 runner.state.prep_complete = Some(false);
14018 runner.state.config.graph.timeout_resource_wait = 30;
14019 runner.state.save().unwrap();
14020 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14021 assert_eq!(
14022 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14023 0
14024 );
14025 assert!(
14026 tokio::time::timeout(Duration::from_millis(100), runner.execute())
14027 .await
14028 .is_err()
14029 );
14030 let disk = RunState::load(&runner.state.id).unwrap();
14031 assert!(disk.driver_exited);
14032 assert_eq!(disk.driver_pid, None);
14033 assert!(disk.status.resumable());
14034 assert!(runner.state.driver_exited);
14035 assert_eq!(runner.state.driver_pid, None);
14036 runner.state.save().unwrap();
14037 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
14038 }
14039
14040 #[tokio::test]
14041 #[cfg(unix)]
14042 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
14043 let tmp = tempfile::tempdir().unwrap();
14044 let mut runner = registered_runner(tmp.path()).await;
14045 let winner = runner.state.candidates[0].clone();
14046 let cache = tmp.path().join("cache");
14047 let started = tmp.path().join("pre-gate-started");
14048 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
14049 runner.state.config.verify.gate =
14050 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
14051 let home = crate::run::home();
14052 let owner =
14053 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
14054 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
14055 crate::cache::AcquireOutcome::Acquired(g) => g,
14056 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
14057 };
14058 let mut gate = Box::pin(runner.run_gate(&winner));
14059 assert!(
14060 tokio::time::timeout(Duration::from_millis(150), &mut gate)
14061 .await
14062 .is_err()
14063 );
14064 assert!(!started.exists(), "pre-gate must wait for the cache too");
14065 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
14069 let editor = crate::repo_lock::SeatLease::checkout_with(
14070 &winner.worktree,
14071 true,
14072 crate::config::test_coordination().as_ref(),
14073 &wait,
14074 )
14075 .await
14076 .unwrap();
14077 drop(editor);
14078 guard.release();
14079 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14080 .await
14081 .unwrap()
14082 .unwrap();
14083 drop(gate);
14084 assert!(outcomes.iter().all(CommandOutcome::ok));
14085 assert!(started.exists());
14086 }
14087
14088 #[tokio::test]
14089 #[cfg(unix)]
14090 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
14091 use std::os::fd::AsRawFd as _;
14092
14093 ask_test_home();
14094 let tmp = tempfile::tempdir().unwrap();
14095 let repo = tmp.path().join("repo");
14096 std::fs::create_dir_all(&repo).unwrap();
14097 init_repo(&repo);
14098 let common = repo.join(".git");
14099 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14100 let mut runner = runner_at(RunStatus::Prep);
14101 runner.state.repo = repo.clone();
14102 runner.state.config.coordination = crate::config::test_coordination();
14103 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14104 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14105 runner.state.config.graph.advise = false;
14106 runner.roles.implementers = vec![spec("implement")];
14107 runner.roles.judges = vec![spec("judge")];
14108 runner.prep().await.unwrap();
14109 let winner = runner.state.candidates[0].clone();
14110 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
14113 let completed = tmp.path().join("seat-completed");
14114 runner.roles.implementers[0].command = vec![
14115 "sh".into(),
14116 "-c".into(),
14117 format!("touch '{}'; echo completed", completed.display()),
14118 ];
14119 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
14120 assert_eq!(
14121 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14122 0
14123 );
14124 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14125 let mut implement = Box::pin(runner.implement());
14126 tokio::select! {
14127 result = &mut implement => panic!("rescue must wait: {result:?}"),
14128 result = tokio::time::timeout(Duration::from_secs(10), async {
14129 while !completed.exists() {
14130 tokio::time::sleep(Duration::from_millis(20)).await;
14131 }
14132 }) => result.unwrap(),
14133 }
14134 assert!(
14135 tokio::time::timeout(Duration::from_millis(200), &mut implement)
14136 .await
14137 .is_err()
14138 );
14139 assert_eq!(
14140 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14141 before
14142 );
14143 assert_eq!(
14144 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14145 0
14146 );
14147 tokio::time::timeout(Duration::from_secs(10), &mut implement)
14148 .await
14149 .unwrap()
14150 .unwrap();
14151 drop(implement);
14152 assert_ne!(
14153 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14154 before
14155 );
14156 assert!(git::is_clean(&winner.worktree).await.unwrap());
14157
14158 let started = tmp.path().join("pre-gate-started");
14159 let proceed = tmp.path().join("pre-gate-proceed");
14160 let checked = tmp.path().join("gate-ran");
14161 runner.state.config.verify.pre_gate = vec![format!(
14162 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14163 started.display(),
14164 proceed.display()
14165 )];
14166 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
14167 assert_eq!(
14168 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14169 0
14170 );
14171 let mut gate = Box::pin(runner.run_gate(&winner));
14172 assert!(
14173 tokio::time::timeout(Duration::from_millis(200), &mut gate)
14174 .await
14175 .is_err()
14176 );
14177 assert!(!started.exists());
14178 assert!(!checked.exists());
14179 assert!(git::is_clean(&winner.worktree).await.unwrap());
14180 assert_eq!(
14181 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
14182 0
14183 );
14184 tokio::select! {
14185 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
14186 result = tokio::time::timeout(Duration::from_secs(10), async {
14187 while !started.exists() {
14188 tokio::time::sleep(Duration::from_millis(20)).await;
14189 }
14190 }) => result.unwrap(),
14191 }
14192 assert_eq!(
14193 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14194 -1,
14195 "mechanical edits must hold the real lock"
14196 );
14197 std::fs::write(proceed, "ready").unwrap();
14198 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
14199 .await
14200 .unwrap()
14201 .unwrap();
14202 drop(gate);
14203 assert!(outcomes.iter().all(CommandOutcome::ok));
14204 assert!(checked.exists());
14205 assert!(runner.state.pre_gate_commit.is_some());
14206 assert!(git::is_clean(&winner.worktree).await.unwrap());
14207 assert_eq!(
14208 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14209 0,
14210 "gate completion must release coordination"
14211 );
14212 drop(outsider);
14213 std::fs::remove_file(common.join("updater.lock")).unwrap();
14214 std::fs::create_dir(common.join("updater.lock")).unwrap();
14215 std::fs::remove_file(&started).unwrap();
14216 std::fs::remove_file(&checked).unwrap();
14217 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14218 let error = runner.run_gate(&winner).await.unwrap_err();
14219 assert!(
14220 error
14221 .to_string()
14222 .contains("acquire mandatory repository lock")
14223 );
14224 assert!(
14225 !started.exists(),
14226 "failed acquisition must prevent pre-gate"
14227 );
14228 assert!(!checked.exists(), "failed acquisition must prevent gate");
14229 assert!(git::is_clean(&winner.worktree).await.unwrap());
14230 assert_eq!(
14231 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14232 head
14233 );
14234 }
14235
14236 #[tokio::test]
14237 #[cfg(unix)]
14238 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
14239 use std::os::fd::AsRawFd as _;
14240 use std::time::Duration;
14241
14242 ask_test_home();
14243 let tmp = tempfile::tempdir().unwrap();
14244 let repo = tmp.path().join("repo");
14245 std::fs::create_dir_all(&repo).unwrap();
14246 init_repo(&repo);
14247 let common = repo.join(".git");
14248 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14249 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
14250 assert_eq!(
14251 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14252 0
14253 );
14254 let mut runner = runner_at(RunStatus::Prep);
14255 runner.state.repo = repo.clone();
14256 runner.state.config.coordination = crate::config::test_coordination();
14257 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14258 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14259 runner.state.config.graph.advise = true;
14260 runner.state.config.graph.advisors = 1;
14261 runner.roles.implementers = vec![spec("implement")];
14262 runner.roles.judges = vec![spec("judge")];
14263 let root = runner.state.worktree_root();
14264 let started = tmp.path().join("setup-started");
14265 let proceed = tmp.path().join("setup-proceed");
14266 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14267 run: Some(format!(
14268 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
14269 started.display(),
14270 proceed.display()
14271 )),
14272 ..Default::default()
14273 }];
14274 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
14275 let config = git::git(&repo, &["config", "--local", "--list"])
14276 .await
14277 .unwrap();
14278 let mut prep = Box::pin(runner.prep());
14279 assert!(
14280 tokio::time::timeout(Duration::from_millis(200), &mut prep)
14281 .await
14282 .is_err()
14283 );
14284 assert!(
14285 !root.exists(),
14286 "contention must prevent even checkout creation"
14287 );
14288 assert!(!started.exists(), "contention must prevent setup");
14289 assert_eq!(
14290 git::git(&repo, &["branch", "--list"]).await.unwrap(),
14291 branches
14292 );
14293 assert_eq!(
14294 git::git(&repo, &["config", "--local", "--list"])
14295 .await
14296 .unwrap(),
14297 config
14298 );
14299 drop(lock);
14300 tokio::select! {
14301 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
14302 result = tokio::time::timeout(Duration::from_secs(10), async {
14303 while !started.exists() {
14304 tokio::time::sleep(Duration::from_millis(20)).await;
14305 }
14306 }) => result.unwrap(),
14307 }
14308 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
14309 assert_eq!(
14310 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14311 -1
14312 );
14313 assert!(
14314 tokio::time::timeout(
14315 Duration::from_millis(100),
14316 crate::repo_lock::SeatLease::for_checkout(
14317 &repo,
14318 true,
14319 crate::config::test_coordination().as_ref()
14320 )
14321 )
14322 .await
14323 .is_err(),
14324 "preparation must also exclude local editors"
14325 );
14326 std::fs::write(proceed, "ready").unwrap();
14327 tokio::time::timeout(Duration::from_secs(10), &mut prep)
14328 .await
14329 .unwrap()
14330 .unwrap();
14331 drop(prep);
14332 assert_eq!(runner.state.prep_complete, Some(true));
14333 assert!(root.join("judge-1").exists());
14334 assert!(root.join("advisor-1").exists());
14335 assert_eq!(
14336 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14337 0
14338 );
14339 }
14340
14341 #[tokio::test]
14342 #[cfg(unix)]
14343 async fn preparation_lock_failure_prevents_repository_changes() {
14344 ask_test_home();
14345 let tmp = tempfile::tempdir().unwrap();
14346 let repo = tmp.path().join("repo");
14347 std::fs::create_dir_all(&repo).unwrap();
14348 init_repo(&repo);
14349 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14350 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14353 let mut runner = runner_at(RunStatus::Prep);
14354 runner.state.repo = repo.clone();
14355 runner.state.config.coordination = crate::config::test_coordination();
14356 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14357 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14358 runner.roles.implementers = vec![spec("implement")];
14359 let error = runner.prep().await.unwrap_err();
14360 assert!(
14361 error
14362 .to_string()
14363 .contains("acquire mandatory repository lock")
14364 );
14365 assert!(!runner.state.worktree_root().exists());
14366 assert!(runner.state.candidates.is_empty());
14367 assert!(!repo.join(".git/config.worktree").exists());
14368 assert_eq!(
14369 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14370 "* main"
14371 );
14372 }
14373
14374 #[tokio::test]
14375 async fn preparation_records_creation_intent_before_git_can_fail() {
14376 ask_test_home();
14377 let tmp = tempfile::tempdir().unwrap();
14378 let repo = tmp.path().join("repo");
14379 std::fs::create_dir_all(&repo).unwrap();
14380 init_repo(&repo);
14381 let mut runner = runner_at(RunStatus::Prep);
14382 runner.state.repo = repo.clone();
14383 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14384 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14385 runner.state.config.blind.commit_msg_hook = false;
14386 runner.roles.implementers = vec![spec("one")];
14387 let label = blind::assign_labels(1, runner.state.seed)[0];
14388 let branch = runner.state.branch_for(label);
14389 git::git(&repo, &["branch", &branch]).await.unwrap();
14392 assert!(runner.prep().await.is_err());
14393 let saved = RunState::load(&runner.state.id).unwrap();
14394 assert!(saved.candidates.is_empty());
14395 let event = saved
14396 .events
14397 .iter()
14398 .find(|e| e.node == "prep-intent")
14399 .unwrap();
14400 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14401 assert_eq!(intent["branch"], branch);
14402 assert_eq!(intent["start"], runner.state.base_commit);
14403 assert_eq!(
14404 intent["worktree"],
14405 runner
14406 .state
14407 .worktree_root()
14408 .join(format!("cand-{label}"))
14409 .to_string_lossy()
14410 .as_ref()
14411 );
14412 assert_eq!(intent["picks"], serde_json::json!([]));
14413 }
14414
14415 #[tokio::test]
14416 async fn invalid_pinned_sources_create_no_partial_worktrees() {
14417 ask_test_home();
14418 let tmp = tempfile::tempdir().unwrap();
14419 let repo = tmp.path().join("repo");
14420 std::fs::create_dir_all(&repo).unwrap();
14421 init_repo(&repo);
14422 let mut runner = runner_at(RunStatus::Prep);
14423 runner.state.repo = repo.clone();
14424 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14425 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14426 runner.state.sources.pick_commits.push(refs::Input {
14427 token: "missing".into(),
14428 sha: "0".repeat(40),
14429 provenance: refs::Provenance::ExplicitCli,
14430 });
14431 assert!(runner.prep().await.is_err());
14432 assert!(runner.state.candidates.is_empty());
14433 assert!(!runner.state.worktree_root().exists());
14434 }
14435
14436 #[tokio::test]
14437 async fn prose_and_legacy_inference_never_import_into_candidates() {
14438 ask_test_home();
14439 let tmp = tempfile::tempdir().unwrap();
14440 let repo = tmp.path().join("repo");
14441 std::fs::create_dir_all(&repo).unwrap();
14442 init_repo(&repo);
14443 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14444 git::git(&repo, &["checkout", "-b", "magi/history/A"])
14445 .await
14446 .unwrap();
14447 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14448 git::git(&repo, &["add", "."]).await.unwrap();
14449 git::git(&repo, &["commit", "-m", "historical implementation"])
14450 .await
14451 .unwrap();
14452 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14453 git::git(&repo, &["checkout", "main"]).await.unwrap();
14454 for instruction in [
14455 format!("Example: magi/history/A at {source}"),
14456 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14457 format!("Never import magi/history/A or {source}"),
14458 format!("Conversation answer: I mentioned {source} only as background"),
14459 ] {
14460 let mut runner = runner_at(RunStatus::Prep);
14461 runner.state.repo = repo.clone();
14462 runner.state.base_commit = base.clone();
14463 runner.state.instruction = instruction.clone();
14464 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14465 runner.state.config.blind.commit_msg_hook = false;
14466 runner.state.seeds = vec![refs::Seed {
14467 token: "magi/history/A".into(),
14468 sha: source.clone(),
14469 kind: refs::SeedKind::Unmerged,
14470 branch: true,
14471 detail: String::new(),
14472 }];
14473 runner.roles.implementers = vec![spec("one")];
14474 assert_eq!(seeded_instruction(&runner.state), instruction);
14475 let mut legacy = serde_json::to_value(&runner.state).unwrap();
14477 legacy.as_object_mut().unwrap().remove("sources");
14478 runner.state = serde_json::from_value(legacy).unwrap();
14479 assert_eq!(runner.state.sources, refs::Sources::default());
14480 runner.state.save().unwrap();
14481 runner.state = RunState::load(&runner.state.id).unwrap();
14482 runner.prep().await.unwrap();
14483 for candidate in &runner.state.candidates {
14484 assert_eq!(
14485 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14486 base
14487 );
14488 assert!(!candidate.worktree.join("unwanted").exists());
14489 }
14490 }
14491 }
14492
14493 #[tokio::test]
14494 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14495 ask_test_home();
14496 let tmp = tempfile::tempdir().unwrap();
14497 let repo = tmp.path().join("repo");
14498 std::fs::create_dir_all(&repo).unwrap();
14499 init_repo(&repo);
14500 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14501 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14502 std::fs::write(repo.join("referenced"), "required commit").unwrap();
14503 git::git(&repo, &["add", "."]).await.unwrap();
14504 git::git(&repo, &["commit", "-m", "required seed"])
14505 .await
14506 .unwrap();
14507 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14508 git::git(&repo, &["checkout", "main"]).await.unwrap();
14509 let mut runner = runner_at(RunStatus::Prep);
14510 runner.state.repo = repo.clone();
14511 runner.state.base_commit = base.clone();
14512 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14513 runner.state.config.graph.advise = false;
14514 runner.state.config.blind.commit_msg_hook = false;
14515 runner.state.sources =
14516 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14517 .await
14518 .unwrap();
14519 runner.roles.implementers = vec![spec("one"), spec("two")];
14520 runner.roles.judges = vec![spec("judge")];
14521 let label = blind::assign_labels(2, runner.state.seed)[0];
14522 let branch = runner.state.branch_for(label);
14523 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14524 git::worktree_add_branch(&repo, &worktree, &branch, &base)
14525 .await
14526 .unwrap();
14527 runner.state.candidates.push(Candidate {
14528 index: 0,
14529 label,
14530 agent: "one".into(),
14531 branch,
14532 worktree,
14533 summary: String::new(),
14534 stat: String::new(),
14535 files: 0,
14536 commits: 0,
14537 empty: false,
14538 failed: None,
14539 verified_noop: None,
14540 duration_ms: 0,
14541 folded: false,
14542 });
14543 runner.state.save().unwrap();
14544 runner.state = RunState::load(&runner.state.id).unwrap();
14546 assert_eq!(runner.state.prep_complete, Some(false));
14547 runner.prep().await.unwrap();
14548 assert_eq!(runner.state.prep_complete, Some(true));
14549 assert_eq!(runner.state.candidates.len(), 2);
14550 for candidate in &runner.state.candidates {
14551 assert_eq!(
14552 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14553 git::tree_of(&repo, &seed).await.unwrap()
14554 );
14555 assert_eq!(
14556 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14557 "required commit"
14558 );
14559 }
14560 assert!(runner.state.worktree_root().join("judge-1").exists());
14561 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14562 runner.prep().await.unwrap();
14563 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14564
14565 runner.state.prep_complete = Some(false);
14566 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14567 run: Some(format!(
14568 "test -f '{}'",
14569 tmp.path().join("dependency-ready").display()
14570 )),
14571 ..Default::default()
14572 }];
14573 assert!(runner.prep().await.is_err());
14574 assert!(runner.state.prep_failure.is_some());
14575 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14576 runner.state = RunState::load(&runner.state.id).unwrap();
14577 assert!(
14578 runner.prep().await.is_err(),
14579 "correcting the cause alone must not auto-retry"
14580 );
14581 let failed_at = runner.state.events.last().unwrap().at;
14582 assert!(
14583 runner.retry_preparation(failed_at).is_err(),
14584 "old approvals cannot unlock the hold"
14585 );
14586 let mut task = crate::queue::Task::new(
14587 "retry".into(),
14588 "retry setup".into(),
14589 repo.clone(),
14590 crate::queue::Source::Human,
14591 );
14592 task.start(runner.state.id.clone());
14593 task.hold_machine(Some("setup failed".into()));
14594 task.release_operator("cli", None);
14595 runner
14596 .retry_preparation(task.resumption.as_ref().unwrap().at)
14597 .unwrap();
14598 runner.prep().await.unwrap();
14599 assert!(runner.state.prep_failure.is_none());
14600 assert_eq!(runner.state.prep_complete, Some(true));
14601 assert_eq!(runner.state.candidates.len(), 2);
14602 assert!(runner.state.worktree_root().join("judge-1").exists());
14603 for candidate in &runner.state.candidates {
14604 assert_eq!(
14605 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14606 git::tree_of(&repo, &seed).await.unwrap()
14607 );
14608 }
14609 }
14610
14611 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14612 let mut heads = Vec::new();
14613 for candidate in candidates {
14614 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14615 }
14616 heads
14617 }
14618
14619 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14620 ask_test_home();
14621 let tmp = tempfile::tempdir().unwrap();
14622 let repo = tmp.path().join("repo");
14623 std::fs::create_dir(&repo).unwrap();
14624 init_repo(&repo);
14625 let mut runner = runner_at(RunStatus::Reviewing);
14626 runner.state.repo = repo.clone();
14627 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14628 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14629 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14630 let branch = format!("candidate-{label}");
14631 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14632 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14633 .await
14634 .unwrap();
14635 if label == 'B' {
14636 std::fs::write(worktree.join("change"), "winner").unwrap();
14637 git::rescue_commit(&worktree, "candidate change")
14638 .await
14639 .unwrap();
14640 }
14641 runner.state.candidates.push(Candidate {
14642 index,
14643 label,
14644 agent: "alpha".into(),
14645 branch,
14646 worktree,
14647 summary: "completed reply".into(),
14648 stat: String::new(),
14649 files: usize::from(label == 'B'),
14650 commits: usize::from(label == 'B'),
14651 empty: label == 'A',
14652 failed: None,
14653 verified_noop: None,
14654 duration_ms: 1,
14655 folded: false,
14656 });
14657 }
14658 runner.state.judge_skipped = true;
14659 runner.state.tally = Some(Tally {
14660 first_choice: BTreeMap::new(),
14661 borda: BTreeMap::new(),
14662 winner: 'B',
14663 rankings: 0,
14664 unanimous_initial: false,
14665 deliberated: false,
14666 changed_votes: 0,
14667 unanimous_final: false,
14668 tie_break: None,
14669 judges: 0,
14670 present: 0,
14671 quorum: 0,
14672 met_quorum: true,
14673 uncontested: Some("only B produced a change".into()),
14674 });
14675 runner.state.save().unwrap();
14676 (tmp, runner)
14677 }
14678
14679 #[tokio::test]
14680 #[cfg(unix)]
14681 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14682 for (solo, failed_checkpoint) in
14683 [(true, false), (false, false), (true, true), (false, true)]
14684 {
14685 let (_tmp, mut runner) = preservation_resume_fixture().await;
14686 if solo {
14687 runner.state.candidates.retain(|c| c.label == 'A');
14688 }
14689 runner.state.candidates[0].empty = false;
14690 runner.state.candidates[0].summary.clear();
14691 runner.state.tally = None;
14692 runner.state.judge_skipped = false;
14693 runner.state.status = RunStatus::Implementing;
14694 runner.state.config.disk.min_free_bytes = u64::MAX;
14695 let mut implementer = spec("alpha");
14696 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14697 runner.roles.implementers = vec![implementer];
14698 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14699 if failed_checkpoint {
14700 std::fs::create_dir_all(&patch).unwrap();
14701 }
14702 runner.state.save().unwrap();
14703 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14704 runner.state = RunState::load(&runner.state.id).unwrap();
14705 assert_eq!(runner.state.status, RunStatus::Blocked);
14706 assert!(runner.state.interrupted_implementations.contains(&'A'));
14707 assert_eq!(runner.state.candidates[0].commits, 0);
14708 assert!(!runner.state.candidates[0].empty);
14709 assert!(runner.state.candidates[0].failed.is_none());
14710 if failed_checkpoint {
14711 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14712 std::fs::remove_dir(patch).unwrap();
14713 }
14714 runner.state.resource_block = None;
14715 runner.state.config.disk.min_free_bytes = 0;
14716 runner.roles.implementers[0].command = vec![
14717 "sh".into(),
14718 "-c".into(),
14719 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14720 ];
14721 runner.implement().await.unwrap();
14722 assert!(runner.state.interrupted_implementations.is_empty());
14723 assert!(runner.state.candidate_preservation.is_empty());
14724 assert_eq!(runner.state.candidates[0].commits, 1);
14725 assert!(!runner.state.candidates[0].empty);
14726 assert!(
14727 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14728 .unwrap()
14729 .contains("retried empty interrupted seat")
14730 );
14731 if !solo {
14732 assert_eq!(runner.state.candidates[1].commits, 1);
14733 }
14734 }
14735 }
14736
14737 #[tokio::test]
14738 #[cfg(unix)]
14739 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14740 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14741 let (_tmp, mut runner) = preservation_resume_fixture().await;
14742 runner.state.candidates.retain(|c| c.label == 'A');
14743 runner.state.candidates[0].empty = false;
14744 runner.state.candidates[0].summary.clear();
14745 runner.state.tally = None;
14746 runner.state.judge_skipped = false;
14747 runner.state.status = RunStatus::Implementing;
14748 runner.state.config.disk.min_free_bytes = u64::MAX;
14751 let wt = runner.state.candidates[0].worktree.clone();
14752 std::fs::write(
14753 wt.join("interrupted-source"),
14754 "retain implementation work\n",
14755 )
14756 .unwrap();
14757 if committed {
14758 git::rescue_commit(&wt, "implementation checkpoint")
14759 .await
14760 .unwrap();
14761 }
14762 let lock = git::git(
14763 &wt,
14764 &[
14765 "rev-parse",
14766 "--path-format=absolute",
14767 "--git-path",
14768 "index.lock",
14769 ],
14770 )
14771 .await
14772 .unwrap();
14773 if block_rescue {
14774 std::fs::write(&lock, "fixture-owned lock").unwrap();
14775 }
14776 let mut implementer = spec("alpha");
14777 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14778 runner.roles.implementers = vec![implementer];
14779 runner.state.save().unwrap();
14780 let result = runner.implement().await;
14781 assert_eq!(result.is_err(), block_rescue);
14782 assert_eq!(
14783 runner.state.resource_block.as_ref().unwrap().kind,
14784 "disk pressure"
14785 );
14786 assert!(
14787 runner
14788 .state
14789 .events
14790 .iter()
14791 .any(|e| e.message.contains("agent execution interrupted"))
14792 );
14793 assert!(
14794 runner
14795 .state
14796 .dir()
14797 .join("artifacts/candidate-A-recovery/staged.patch")
14798 .exists()
14799 );
14800 if block_rescue {
14801 let saved = RunState::load(&runner.state.id).unwrap();
14802 assert!(saved.candidate_preservation.contains_key(&'A'));
14803 assert!(!saved.candidates[0].empty);
14804 assert_eq!(
14805 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14806 "retain implementation work\n"
14807 );
14808 std::fs::remove_file(lock).unwrap();
14809 runner.state = saved;
14810 } else {
14811 assert_eq!(runner.state.status, RunStatus::Blocked);
14812 assert_eq!(runner.state.candidates[0].commits, 1);
14813 }
14814 runner.roles.implementers.clear();
14817 runner.state.config.disk.min_free_bytes = 0;
14818 runner.state.resource_block = None;
14819 runner.implement().await.unwrap();
14820 assert!(runner.state.candidate_preservation.is_empty());
14821 assert_eq!(runner.state.candidates[0].commits, 1);
14822 assert!(!runner.state.candidates[0].empty);
14823 assert!(git::is_clean(&wt).await.unwrap());
14824 assert!(
14825 crate::run::read_artifact(&runner.state, "cand-A.patch")
14826 .unwrap()
14827 .contains("retain implementation work")
14828 );
14829 }
14830 }
14831
14832 #[tokio::test]
14833 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14834 let (_tmp, mut runner) = preservation_resume_fixture().await;
14835 runner.implement().await.unwrap();
14837 assert!(runner.state.candidate_preservation.is_empty());
14838 assert!(
14839 !runner
14840 .state
14841 .dir()
14842 .join("artifacts/candidate-A-recovery")
14843 .exists()
14844 );
14845 runner.fold_losers().await.unwrap();
14846 assert!(!runner.state.candidates[0].worktree.exists());
14847 for schema in [14, crate::run::SCHEMA] {
14848 runner.state.schema = schema;
14849 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14850 runner.state.save().unwrap();
14851 runner.state = RunState::load(&runner.state.id).unwrap();
14852 runner.implement().await.unwrap();
14853 assert!(runner.state.candidate_preservation.is_empty());
14854 assert!(runner.state.candidates[0].folded);
14855 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14856 }
14857 }
14858
14859 #[tokio::test]
14860 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14861 let (_tmp, mut runner) = preservation_resume_fixture().await;
14862 runner.state.candidates.retain(|c| c.label == 'A');
14863 runner.state.tally = None;
14864 runner.state.judge_skipped = false;
14865 let candidate = &mut runner.state.candidates[0];
14866 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14867 candidate.failed = Some("deployment validation unavailable".into());
14868 std::fs::write(
14869 candidate.worktree.join("repair"),
14870 "future launcher policy\n",
14871 )
14872 .unwrap();
14873 runner
14874 .state
14875 .candidate_preservation
14876 .insert('A', "awaiting persistence".into());
14877 runner.persist_candidates().await.unwrap();
14878 let candidate = &runner.state.candidates[0];
14879 assert!(!candidate.empty);
14880 assert_eq!(candidate.commits, 1);
14881 assert_eq!(candidate.files, 1);
14882 assert!(candidate.failed.is_none());
14883 assert!(candidate.summary.contains("EPERM"));
14884 assert!(
14885 crate::run::read_artifact(&runner.state, "cand-A.patch")
14886 .unwrap()
14887 .contains("future launcher policy")
14888 );
14889 assert!(runner.state.candidate_preservation.is_empty());
14890 }
14891
14892 #[tokio::test]
14893 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14894 let (_tmp, mut runner) = preservation_resume_fixture().await;
14895 let wt = runner.state.candidates[0].worktree.clone();
14896 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14897 runner.state.schema = 14;
14898 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14899 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14900 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14901 for _ in 0..2 {
14902 assert!(runner.implement().await.is_err());
14903 assert_eq!(runner.state.status, RunStatus::Blocked);
14904 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14905 assert!(
14906 runner.state.candidates[0].empty,
14907 "old verdict must remain unusable"
14908 );
14909 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14910 assert!(runner.state.judge_skipped);
14911 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14912 assert!(
14913 crate::run::read_artifact(&runner.state, "cand-A.patch")
14914 .unwrap()
14915 .contains("owned edits")
14916 );
14917 assert!(wt.exists());
14918 assert_eq!(
14919 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14920 .await
14921 .unwrap(),
14922 1
14923 );
14924 runner.state = RunState::load(&runner.state.id).unwrap();
14925 }
14926 }
14927
14928 #[tokio::test]
14929 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14930 let (_tmp, mut runner) = preservation_resume_fixture().await;
14931 let wt = runner.state.candidates[0].worktree.clone();
14932 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14933 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14934 let index_lock = PathBuf::from(
14935 git::git(
14936 &wt,
14937 &[
14938 "rev-parse",
14939 "--path-format=absolute",
14940 "--git-path",
14941 "index.lock",
14942 ],
14943 )
14944 .await
14945 .unwrap(),
14946 );
14947 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14948 runner.state.schema = 14;
14949 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14950 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14951 assert!(runner.implement().await.is_err());
14952 assert!(runner.after_implement().is_err());
14953 assert!(runner.fold_losers().await.is_err());
14954 assert_eq!(
14955 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14956 "failed seat edits"
14957 );
14958 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14959 runner.state = RunState::load(&runner.state.id).unwrap();
14960 std::fs::remove_file(index_lock).unwrap();
14961 assert!(runner.implement().await.is_err());
14963 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14964 assert!(runner.state.candidates[0].empty);
14965 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14966 assert!(
14967 crate::run::read_artifact(&runner.state, "cand-A.patch")
14968 .unwrap()
14969 .contains("failed seat edits")
14970 );
14971 assert!(wt.exists());
14972 }
14973
14974 #[tokio::test]
14975 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14976 let (_tmp, mut runner) = preservation_resume_fixture().await;
14977 let repo = runner.state.repo.clone();
14978 let common = git::git(
14979 &repo,
14980 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14981 )
14982 .await
14983 .unwrap();
14984 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14985 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14986 assert!(runner.fold_losers().await.is_err());
14987 assert!(!runner.state.candidates[0].worktree.exists());
14988 assert!(!runner.state.candidates[0].folded);
14989 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14990 runner.state = RunState::load(&runner.state.id).unwrap();
14991 std::fs::remove_file(branch_lock).unwrap();
14992 let judge = runner.state.worktree_root().join("judge-1");
14993 git::worktree_add_detached(&repo, &judge, "main")
14994 .await
14995 .unwrap();
14996 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14997 runner.roles.judges = vec![spec("judge")];
14998 assert!(runner.fold_losers().await.is_err());
14999 runner.state = RunState::load(&runner.state.id).unwrap();
15000 assert!(
15001 runner.state.candidates[0].folded,
15002 "candidate progress saved before judge failure"
15003 );
15004 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
15005 assert!(judge.join("uncommitted").exists());
15006 std::fs::remove_file(judge.join("uncommitted")).unwrap();
15007 runner.fold_losers().await.unwrap();
15008 assert!(!judge.exists());
15009 runner.fold_losers().await.unwrap();
15010 }
15011
15012 #[tokio::test]
15013 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
15014 let (_tmp, mut runner) = preservation_resume_fixture().await;
15015 runner.state.candidates.retain(|c| c.label == 'B');
15016 runner.state.tally = None;
15017 runner.state.judge_skipped = false;
15018 runner.state.prep_complete = Some(true);
15019 runner.state.config.graph.advise = false;
15020 runner.state.config.graph.selected_implementer = Some("alpha".into());
15021 let wt = runner.state.candidates[0].worktree.clone();
15022 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
15023 runner
15024 .state
15025 .candidate_preservation
15026 .insert('B', "awaiting persistence".into());
15027 let index_lock = git::git(
15028 &wt,
15029 &[
15030 "rev-parse",
15031 "--path-format=absolute",
15032 "--git-path",
15033 "index.lock",
15034 ],
15035 )
15036 .await
15037 .unwrap();
15038 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15039 runner.state.save().unwrap();
15040 assert!(runner.execute_graph().await.is_err());
15041 assert_eq!(runner.state.status, RunStatus::Blocked);
15042 assert!(runner.state.candidate_preservation.contains_key(&'B'));
15043 std::fs::remove_file(index_lock).unwrap();
15044 for _ in 0..2 {
15047 runner.state = RunState::load(&runner.state.id).unwrap();
15048 runner.pause.park();
15049 runner.execute_graph().await.unwrap();
15050 assert_eq!(runner.state.status, RunStatus::Implementing);
15051 assert!(runner.state.candidate_preservation.is_empty());
15052 assert_eq!(runner.state.candidates[0].commits, 2);
15053 assert!(runner.state.tally.is_none());
15054 }
15055 runner.implement().await.unwrap();
15056 assert_eq!(runner.state.status, RunStatus::Judging);
15057 runner.state.status = RunStatus::Blocked;
15059 runner.state.candidates[0].failed = Some("implementation failed".into());
15060 runner.state.save().unwrap();
15061 let error = runner.execute_graph().await.unwrap_err();
15062 assert!(
15063 error
15064 .to_string()
15065 .contains("selected implementation is held")
15066 );
15067 }
15068
15069 #[tokio::test]
15070 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
15071 let (_tmp, mut runner) = preservation_resume_fixture().await;
15072 let repo = runner.state.repo.clone();
15073 let candidate = runner.state.candidates[0].clone();
15074 let registered = git::worktree_holding(&repo, &candidate.branch)
15077 .await
15078 .unwrap()
15079 .unwrap();
15080 assert_eq!(
15081 registered.canonicalize().unwrap(),
15082 candidate.worktree.canonicalize().unwrap()
15083 );
15084 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15085 assert_eq!(
15086 git::worktree_holding(&repo, &candidate.branch)
15087 .await
15088 .unwrap(),
15089 Some(registered)
15090 );
15091 assert!(
15092 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
15093 "the stale registration initially keeps the branch checked out"
15094 );
15095 runner.fold_losers().await.unwrap();
15096 assert!(runner.state.candidates[0].folded);
15097 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15098 assert_eq!(
15099 git::worktree_holding(&repo, &candidate.branch)
15100 .await
15101 .unwrap(),
15102 None
15103 );
15104 runner.state = RunState::load(&runner.state.id).unwrap();
15105 runner.fold_losers().await.unwrap();
15106 }
15107
15108 #[tokio::test]
15109 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
15110 let (_tmp, mut runner) = preservation_resume_fixture().await;
15111 let repo = runner.state.repo.clone();
15112 let candidate = runner.state.candidates[0].clone();
15113 let winner = runner.state.candidates[1].clone();
15114 let registered = git::worktree_holding(&repo, &candidate.branch)
15115 .await
15116 .unwrap()
15117 .unwrap();
15118 assert_eq!(
15119 registered.canonicalize().unwrap(),
15120 candidate.worktree.canonicalize().unwrap()
15121 );
15122 std::fs::remove_dir_all(&candidate.worktree).unwrap();
15123 assert_eq!(
15124 git::worktree_holding(&repo, &candidate.branch)
15125 .await
15126 .unwrap(),
15127 Some(registered)
15128 );
15129 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
15130
15131 for _ in 0..2 {
15132 fold_run(&mut runner.state, false, &crate::run::home())
15133 .await
15134 .unwrap();
15135 runner.state = RunState::load(&runner.state.id).unwrap();
15136 assert!(runner.state.candidates[0].folded);
15137 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
15138 assert_eq!(
15139 git::worktree_holding(&repo, &candidate.branch)
15140 .await
15141 .unwrap(),
15142 None
15143 );
15144 assert!(!runner.state.candidates[1].folded);
15145 assert!(winner.worktree.exists());
15146 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
15147 }
15148 }
15149
15150 #[tokio::test]
15151 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
15152 ask_test_home();
15153 let tmp = tempfile::tempdir().unwrap();
15154 let repo = tmp.path().join("repo");
15155 std::fs::create_dir(&repo).unwrap();
15156 init_repo(&repo);
15157 let mut runner = runner_at(RunStatus::Implementing);
15158 runner.state.repo = repo.clone();
15159 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15160 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15161 for (index, label) in ['A', 'B'].into_iter().enumerate() {
15162 let branch = format!("candidate-{label}");
15163 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
15164 git::worktree_add_branch(&repo, &wt, &branch, "main")
15165 .await
15166 .unwrap();
15167 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
15168 git::git(&wt, &["add", "README.md"]).await.unwrap();
15169 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
15170 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
15171 runner.state.candidates.push(Candidate {
15172 index,
15173 label,
15174 agent: "alpha".to_owned(),
15175 branch,
15176 worktree: wt,
15177 summary: "completed implementation".to_owned(),
15178 stat: String::new(),
15179 files: 0,
15180 commits: 0,
15181 empty: false,
15182 failed: None,
15183 verified_noop: None,
15184 duration_ms: 42,
15185 folded: false,
15186 });
15187 runner
15188 .state
15189 .candidate_preservation
15190 .insert(label, "awaiting persistence".to_owned());
15191 }
15192 let wt = runner.state.candidates[0].worktree.clone();
15193 let index_lock = git::git(
15194 &wt,
15195 &[
15196 "rev-parse",
15197 "--path-format=absolute",
15198 "--git-path",
15199 "index.lock",
15200 ],
15201 )
15202 .await
15203 .unwrap();
15204 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15205 runner.state.save().unwrap();
15206 assert!(runner.persist_candidates().await.is_err());
15207 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15208 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
15209 assert_eq!(
15210 runner.state.candidates[1].commits, 1,
15211 "other results are still collected"
15212 );
15213 assert!(
15214 !runner.state.candidates[0].empty,
15215 "failed rescue is not no change"
15216 );
15217 assert_eq!(
15218 runner.state.candidates[0].summary,
15219 "completed implementation"
15220 );
15221 assert!(runner.after_implement().is_err());
15222 let home = tmp.path().join("fold-home");
15223 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
15224 assert!(wt.join("new-file").exists());
15225 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
15226 assert!(recovery.join("staged.patch").exists());
15227 assert!(recovery.join("unstaged.patch").exists());
15228 assert_eq!(
15229 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
15230 "untracked edits\n"
15231 );
15232 runner.state.begin_pass();
15233 runner.state.end_pass_parked(ParkCause::Upgrade {
15234 from: "1.0".into(),
15235 to: Some("1.1".into()),
15236 });
15237 runner.state.save().unwrap();
15238 runner.state = RunState::load(&runner.state.id).unwrap();
15239 assert!(matches!(
15240 runner.state.passes.last().unwrap().end,
15241 Some(crate::run::PassEnd::Parked {
15242 cause: ParkCause::Upgrade { .. }
15243 })
15244 ));
15245 std::fs::remove_file(index_lock).unwrap();
15246 runner.implement().await.unwrap();
15248 assert!(runner.state.candidate_preservation.is_empty());
15249 assert!(
15250 runner
15251 .state
15252 .candidates
15253 .iter()
15254 .all(|c| c.commits == 1 && !c.empty)
15255 );
15256 assert!(git::is_clean(&wt).await.unwrap());
15257 assert!(
15258 crate::run::read_artifact(&runner.state, "cand-A.patch")
15259 .unwrap()
15260 .contains("later tracked edits")
15261 );
15262 }
15263
15264 #[tokio::test]
15265 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
15266 ask_test_home();
15267 let tmp = tempfile::tempdir().unwrap();
15268 let repo = tmp.path().join("repo");
15269 std::fs::create_dir(&repo).unwrap();
15270 init_repo(&repo);
15271 let wt = tmp.path().join("candidate");
15272 git::worktree_add_branch(&repo, &wt, "candidate", "main")
15273 .await
15274 .unwrap();
15275 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15276 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
15277 assert!(wt.join("uncommitted").exists());
15278 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15279 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15280 let mut runner = runner_at(RunStatus::Implementing);
15281 runner.state.repo = repo.clone();
15282 runner.state.base_commit = "missing-base".to_owned();
15283 runner.state.candidates.push(Candidate {
15284 index: 0,
15285 label: 'A',
15286 agent: "alpha".into(),
15287 branch: "candidate".into(),
15288 worktree: wt.clone(),
15289 summary: "completed".into(),
15290 stat: String::new(),
15291 files: 0,
15292 commits: 0,
15293 empty: false,
15294 failed: None,
15295 verified_noop: None,
15296 duration_ms: 1,
15297 folded: false,
15298 });
15299 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
15300 assert!(
15301 fold_run(&mut runner.state, true, &tmp.path().join("home"))
15302 .await
15303 .is_err()
15304 );
15305 assert!(!runner.state.candidates[0].folded);
15306 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
15307 std::fs::remove_file(wt.join("uncommitted")).unwrap();
15308 runner
15309 .state
15310 .candidate_preservation
15311 .insert('A', "pending".into());
15312 assert!(runner.persist_candidates().await.is_err());
15313 assert!(!runner.state.candidates[0].empty);
15314 assert!(!runner.state.candidates[0].folded);
15315 assert!(wt.exists());
15316 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15317 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
15318 std::fs::create_dir_all(&patch_path).unwrap();
15319 assert!(
15320 runner.persist_candidates().await.is_err(),
15321 "artifact failure stays pending"
15322 );
15323 assert!(runner.state.candidate_preservation.contains_key(&'A'));
15324 assert!(!runner.state.candidates[0].empty);
15325 std::fs::remove_dir(patch_path).unwrap();
15326 runner.persist_candidates().await.unwrap();
15327 assert!(
15328 runner.state.candidates[0].empty,
15329 "only successful inspection confirms no change"
15330 );
15331 }
15332
15333 fn init_repo(dir: &Path) {
15334 let run = |args: &[&str]| {
15335 let out = std::process::Command::new("git")
15336 .args(args)
15337 .current_dir(dir)
15338 .quiet()
15339 .output()
15340 .expect("spawn git");
15341 assert!(
15342 out.status.success(),
15343 "git {args:?} failed: {}",
15344 String::from_utf8_lossy(&out.stderr)
15345 );
15346 };
15347 run(&["init", "-b", "main"]);
15348 run(&["config", "user.name", "magi test"]);
15349 run(&["config", "user.email", "magi@example.com"]);
15350 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15351 run(&["add", "-A"]);
15352 run(&["commit", "-m", "init"]);
15353 }
15354
15355 fn ask_test_home() {
15363 crate::run::pin_test_home();
15364 }
15365
15366 #[cfg(unix)]
15367 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15368 ask_test_home();
15369 let tmp = tempfile::tempdir().unwrap();
15370 let repo = tmp.path().join("repo");
15371 std::fs::create_dir(&repo).unwrap();
15372 init_repo(&repo);
15373 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15374 let origin = tmp.path().join("origin.git");
15375 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15376 .await
15377 .unwrap();
15378 git::git(
15379 &repo,
15380 &["remote", "add", "origin", origin.to_str().unwrap()],
15381 )
15382 .await
15383 .unwrap();
15384 git::git(&repo, &["checkout", "-b", "fix/retained"])
15385 .await
15386 .unwrap();
15387 std::fs::write(repo.join("retained"), "retained change").unwrap();
15388 git::git(&repo, &["add", "."]).await.unwrap();
15389 git::git(&repo, &["commit", "-m", "retained implementation"])
15390 .await
15391 .unwrap();
15392 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15393 git::git(&repo, &["checkout", "main"]).await.unwrap();
15394 std::fs::write(repo.join("base-change"), "new base").unwrap();
15395 git::git(&repo, &["add", "."]).await.unwrap();
15396 git::git(&repo, &["commit", "-m", "base moved"])
15397 .await
15398 .unwrap();
15399 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15400 git::git(&repo, &["checkout", "fix/retained"])
15401 .await
15402 .unwrap();
15403 let mut runner = runner_at(RunStatus::Voting);
15404 runner.state.repo = repo.clone();
15405 runner.state.base_commit = base;
15406 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15407 runner.state.config.disk.min_free_bytes = 0;
15408 runner.state.config.graph.review_rounds = 1;
15409 runner.state.config.verify.e2e = vec!["true".into()];
15410 runner.state.config.verify.gate = vec!["true".into()];
15411 runner.state.candidates = vec![Candidate {
15412 index: 0,
15413 label: 'A',
15414 agent: "fixture".into(),
15415 branch: "fix/retained".into(),
15416 worktree: repo.clone(),
15417 summary: String::new(),
15418 stat: String::new(),
15419 files: 1,
15420 commits: 1,
15421 empty: false,
15422 failed: None,
15423 verified_noop: None,
15424 duration_ms: 0,
15425 folded: false,
15426 }];
15427 runner.state.tally = Some(Tally {
15428 first_choice: BTreeMap::new(),
15429 borda: BTreeMap::new(),
15430 winner: 'A',
15431 rankings: 0,
15432 unanimous_initial: true,
15433 deliberated: false,
15434 changed_votes: 0,
15435 unanimous_final: true,
15436 tie_break: None,
15437 judges: 0,
15438 present: 0,
15439 quorum: 0,
15440 met_quorum: true,
15441 uncontested: Some("review-only".into()),
15442 });
15443 let mut task = crate::queue::Task::new(
15444 "fixture".into(),
15445 "fixture".into(),
15446 repo,
15447 crate::queue::Source::Human,
15448 );
15449 task.link_run(&runner.state.id);
15450 crate::queue::Queue::open().put(&mut task).unwrap();
15451 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15452 runner.state.base_sync = Some(BaseSync {
15453 tip: String::new(),
15454 behind: 1,
15455 attempts: BASE_SYNC_ROUNDS,
15456 conflict: None,
15457 already_in: None,
15458 });
15459 runner.sync_to_base().await.unwrap();
15460 assert_eq!(runner.state.status, RunStatus::Blocked);
15461 (tmp, runner, head, task.id)
15462 }
15463
15464 #[cfg(unix)]
15465 #[tokio::test]
15466 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15467 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15468 if std::env::var_os(CHILD).is_none() {
15469 let home = tempfile::tempdir().unwrap();
15470 let output = std::process::Command::new(std::env::current_exe().unwrap())
15471 .args([
15472 "--exact",
15473 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15474 "--nocapture",
15475 ])
15476 .env(CHILD, "1")
15477 .env("MAGI_HOME", home.path())
15478 .env("MAGI_WORKER_HOME", home.path())
15479 .env("MAGI_WORKER_ID", "abcdef")
15480 .output()
15481 .unwrap();
15482 assert!(
15483 output.status.success()
15484 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15485 "{}\n{}",
15486 String::from_utf8_lossy(&output.stdout),
15487 String::from_utf8_lossy(&output.stderr)
15488 );
15489 return;
15490 }
15491 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15492 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15493 let home = crate::run::home();
15494 let queue = crate::queue::Queue::open();
15495 let mut owner = queue.get(&task).unwrap();
15496 owner.release_operator("test-release", None);
15497 owner.start(runner.state.id.clone());
15498 queue.put(&mut owner).unwrap();
15499 runner.state.driver_exited = true;
15500 runner.state.task_instruction = Some(owner.instruction.clone());
15501 runner.state.save().unwrap();
15502 let _claim = queue.claim(&task).unwrap();
15503 let reserve = |id: &str| {
15504 let directory = home.join("workers/jobs").join(id);
15505 std::fs::create_dir_all(&directory).unwrap();
15506 let job = crate::worker::Job {
15507 id: id.into(),
15508 sequence: 1,
15509 kind: "task".into(),
15510 key: format!("task:{task}"),
15511 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15512 generation: "fixture-generation".into(),
15513 version: None,
15514 status: crate::worker::Status::Running,
15515 result: None,
15516 pid: Some(std::process::id()),
15517 created_at: 0,
15518 finished_at: None,
15519 };
15520 std::fs::write(
15521 directory.join("job.json"),
15522 serde_json::to_vec(&job).unwrap(),
15523 )
15524 .unwrap();
15525 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15526 };
15527 let _own = reserve("abcdef");
15528 let other = reserve("fedcba");
15529 assert!(
15530 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15531 .await
15532 .is_err()
15533 );
15534 assert!(
15535 RunState::load(&runner.state.id)
15536 .unwrap()
15537 .base_sync_recovery
15538 .authorizations
15539 .is_empty()
15540 );
15541 drop(other);
15542 assert!(crate::daemon::is_working_on(
15543 &home,
15544 &runner.state.id,
15545 Timestamp::now()
15546 ));
15547 assert!(
15548 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15549 );
15550 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15551 .await
15552 .unwrap();
15553 assert!(
15554 RunState::load(&runner.state.id)
15555 .unwrap()
15556 .base_recovery_pending()
15557 );
15558 }
15559
15560 #[cfg(unix)]
15561 #[tokio::test]
15562 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15563 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15564 let home = crate::run::home();
15565 runner.state.driver_pid = Some(std::process::id());
15566 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15567 runner.state.driver_exited = false;
15568 runner.state.begin_pass();
15569 runner.state.save().unwrap();
15570 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15571 driver.record(&runner.state).unwrap();
15572 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15573 drop(driver);
15575 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15576 assert_eq!(
15577 runner.state.liveness_under(&home, false),
15578 crate::run::Liveness::Dead
15579 );
15580 let queue = crate::queue::Queue::open();
15581 let _claim = queue.claim(&task).unwrap();
15582 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15583 .await
15584 .unwrap();
15585 let disk = RunState::load(&runner.state.id).unwrap();
15586 assert!(!disk.driver_exited);
15587 assert!(disk.base_recovery_pending());
15588 }
15589
15590 #[cfg(unix)]
15591 #[tokio::test]
15592 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15593 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15594 if std::env::var_os(CHILD).is_none() {
15595 use std::os::unix::fs::PermissionsExt;
15596 let tmp = tempfile::tempdir().unwrap();
15597 let gh = tmp.path().join("gh");
15598 std::fs::write(&gh, r#"#!/bin/sh
15599case "$1 $2" in
15600 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15601 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15602 *) exit 2 ;;
15603esac
15604"#).unwrap();
15605 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15606 let mut paths = vec![tmp.path().to_path_buf()];
15607 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15608 let output = std::process::Command::new(std::env::current_exe().unwrap())
15609 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15610 .env(CHILD, "1")
15611 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15612 .env("PATH", std::env::join_paths(paths).unwrap())
15613 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15614 .output().unwrap();
15615 assert!(
15616 output.status.success()
15617 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15618 "{}\n{}",
15619 String::from_utf8_lossy(&output.stdout),
15620 String::from_utf8_lossy(&output.stderr)
15621 );
15622 return;
15623 }
15624 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15625 let repo = runner.state.repo.clone();
15626 git::git(&repo, &["checkout", "main"]).await.unwrap();
15627 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15628 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15629 git::git(&repo, &["checkout", "fix/retained"])
15630 .await
15631 .unwrap();
15632 runner.state.pr = Some(crate::run::PrRecord {
15633 url: "https://example.test/project/pull/1".into(),
15634 number: 1,
15635 state: "open".into(),
15636 checks: "unknown".into(),
15637 round: 0,
15638 rounds: 1,
15639 red_at_merge: Vec::new(),
15640 });
15641 runner.sync_to_base().await.unwrap();
15642 assert_eq!(runner.state.status, RunStatus::Blocked);
15643 assert_eq!(
15644 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15645 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15646 );
15647 assert!(runner.state.base_sync_closure_pending());
15648 assert!(crate::handover::can_resume(&runner.state));
15649 let queue = crate::queue::Queue::open();
15650 assert!(!crate::base_sync_recovery::held_task(
15651 &queue,
15652 &queue.get(&task).unwrap()
15653 ));
15654 assert!(runner.state.authorize_base_sync(&task).is_err());
15655 runner.state.base_sync_recovery.stop = None;
15657 runner.state.status = RunStatus::Voting;
15658 runner.state.gate_ran = true;
15659 runner.state.config.graph.review_rounds = 0;
15660 runner.state.save().unwrap();
15661 assert!(crate::handover::can_resume(&runner.state));
15662 runner.state = RunState::load(&runner.state.id).unwrap();
15663 runner.execute_graph().await.unwrap();
15664 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15665 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15666 assert_eq!(
15667 runner.state.base_sync.as_ref().unwrap().attempts,
15668 BASE_SYNC_ROUNDS
15669 );
15670 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15671 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15672 assert!(runner.state.reviews.is_empty());
15673 assert!(runner.state.gate.is_empty());
15674 }
15675
15676 #[cfg(unix)]
15677 #[tokio::test]
15678 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15679 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15680 runner.state.driver_exited = true;
15681 runner.state.save().unwrap();
15682 let home = crate::run::home();
15683 let queue = crate::queue::Queue::open();
15684 let _claim = queue.claim(&task).unwrap();
15685 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15686 assert!(
15687 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15688 .await
15689 .is_err()
15690 );
15691 drop(lease);
15692 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15693 assert!(
15694 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15695 .await
15696 .is_err()
15697 );
15698 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15699 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15700 .await
15701 .unwrap();
15702 let disk = RunState::load(&runner.state.id).unwrap();
15703 assert!(disk.base_recovery_pending());
15704 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15705 assert!(
15706 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15707 .await
15708 .is_err()
15709 );
15710 assert_eq!(
15711 RunState::load(&runner.state.id)
15712 .unwrap()
15713 .base_sync_recovery
15714 .authorizations
15715 .len(),
15716 1
15717 );
15718 let mut owner = queue.get(&task).unwrap();
15719 owner.runs.push("20260101-000000-ffff".into());
15720 queue.put(&mut owner).unwrap();
15721 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15722 .await
15723 .unwrap_err();
15724 assert!(error.to_string().contains("run task ownership changed"));
15725 }
15726
15727 #[cfg(unix)]
15728 #[tokio::test]
15729 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15730 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15731 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15732 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15733 legacy["status"] = serde_json::json!("voting");
15734 runner.state = serde_json::from_value(legacy).unwrap();
15735 runner.state.save().unwrap();
15736 for _ in 0..3 {
15737 runner.execute_graph().await.unwrap();
15738 assert_eq!(runner.state.status, RunStatus::Blocked);
15739 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15740 assert!(!crate::handover::can_resume(&runner.state));
15741 assert_eq!(
15742 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15743 head
15744 );
15745 assert!(runner.state.reviews.is_empty());
15746 assert!(!runner.state.gate_ran);
15747 }
15748 let notice = crate::notices::run_ended(&runner.state).unwrap();
15749 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15750 runner.state.begin_pass();
15751 runner.state.status = RunStatus::Voting;
15752 runner.deliberate().await.unwrap();
15753 runner.vote().await.unwrap();
15754 runner.finish_after_tally().await.unwrap();
15755 assert_eq!(runner.state.status, RunStatus::Blocked);
15756 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15757 assert_eq!(
15758 crate::notices::run_ended(&runner.state).unwrap().since,
15759 notice.since
15760 );
15761 }
15762
15763 #[cfg(unix)]
15764 #[tokio::test]
15765 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15766 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15767 for detail in [
15768 "fix/retained conflicts with origin/main and did not rebase: conflict",
15769 "rebased fix/retained locally but origin refused the push",
15770 "unknown failure",
15771 ] {
15772 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15773 runner.state.record_base_sync_stop(&head);
15774 assert!(runner.state.authorize_base_sync(&task).is_err());
15775 runner.state.status = RunStatus::Voting;
15776 runner.execute_graph().await.unwrap();
15777 assert_eq!(runner.state.status, RunStatus::Blocked);
15778 assert!(!crate::handover::can_resume(&runner.state));
15779 assert_eq!(
15780 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15781 head
15782 );
15783 }
15784 }
15785
15786 #[cfg(unix)]
15787 #[tokio::test]
15788 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15789 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15790 runner.state.authorize_base_sync(&task).unwrap();
15791 assert!(runner.state.consume_base_recovery(&head));
15792 runner.state.save().unwrap();
15793 runner.state = RunState::load(&runner.state.id).unwrap();
15794 runner.execute_graph().await.unwrap();
15795 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15796 assert_eq!(runner.state.status, RunStatus::Blocked);
15797 assert!(!runner.state.consume_base_recovery(&head));
15798 assert!(!crate::handover::can_resume(&runner.state));
15799 runner.state.authorize_base_sync(&task).unwrap();
15800 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15801 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15802 assert_eq!(
15803 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15804 head
15805 );
15806 }
15807
15808 #[cfg(unix)]
15809 async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
15810 let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
15811 runner.state.prep_complete = Some(true);
15813 runner.state.base_sync = None;
15814 runner.state.base_sync_recovery = Default::default();
15815 runner.state.config.graph.gate_fix_rounds = 1;
15816 let mut review = review_round(true, 0, 1, 1, false, true);
15817 review.head = head.clone();
15818 review.verified_head = Some(head.clone());
15819 runner.state.reviews = vec![review];
15820 runner.state.gate = vec![CommandOutcome {
15821 command: "old gate".into(),
15822 code: Some(1),
15823 output_tail: "old failure".into(),
15824 duration_ms: 0,
15825 resource_blocked: false,
15826 }];
15827 runner.state.gate_head = Some(head.clone());
15828 runner.state.gate_ran = true;
15829 runner.state.gate_fixes = vec![crate::run::GateFixRecord {
15830 agent: "fixture".into(),
15831 failed: runner.state.gate.clone(),
15832 notes: "spent".into(),
15833 committed: false,
15834 error: None,
15835 }];
15836 runner.state.land_approval = Some(crate::run::LandApproval {
15837 question: "old approval".into(),
15838 head: head.clone(),
15839 });
15840 runner.state.status = RunStatus::Blocked;
15841 (tmp, runner, head)
15842 }
15843
15844 #[cfg(unix)]
15845 #[tokio::test]
15846 async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
15847 for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
15848 let (tmp, mut runner, old) = resumed_gate_fixture().await;
15849 let phases = tmp.path().join("phases");
15850 runner.state.config.verify.e2e = vec![format!(
15851 "echo e2e >> '{}'; {}",
15852 phases.display(),
15853 if e2e_ok {
15854 "true"
15855 } else {
15856 "echo new-e2e-failure; false"
15857 }
15858 )];
15859 runner.state.config.verify.gate = vec![format!(
15860 "echo gate >> '{}'; {}",
15861 phases.display(),
15862 if gate_ok {
15863 "true"
15864 } else {
15865 "echo new-gate-failure; false"
15866 }
15867 )];
15868 runner.state.save().unwrap();
15869 runner.state = RunState::load(&runner.state.id).unwrap();
15870 runner.sync_to_base().await.unwrap();
15871 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15872 assert_ne!(head, old);
15873 runner.state = RunState::load(&runner.state.id).unwrap();
15875 runner.execute_graph().await.unwrap();
15876 assert_eq!(runner.state.gate_fixes.len(), 1);
15877 assert_eq!(runner.state.reviews.len(), 1);
15878 assert_eq!(runner.state.reviews[0].head, old);
15879 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
15880 assert_eq!(runner.state.verification_history.len(), 1);
15881 let history = &runner.state.verification_history[0];
15882 assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
15883 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15884 assert!(history.archived_at.is_some());
15885 assert_eq!(history.fixes_spent, Some(1));
15886 assert_eq!(history.gate[0].output_tail, "old failure");
15887 let archive_events: Vec<_> = runner
15888 .state
15889 .events
15890 .iter()
15891 .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
15892 .collect();
15893 assert_eq!(archive_events.len(), 1);
15894 assert!(archive_events[0].message.contains(&short(&old)));
15895 assert!(archive_events[0].message.contains(&short(&head)));
15896 assert!(
15897 archive_events[0]
15898 .message
15899 .contains("1 gate-fix round(s) already spent stay spent")
15900 );
15901 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
15902 assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
15903 if e2e_ok {
15904 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15905 assert_eq!(runner.state.gate_allows_progress(), gate_ok);
15906 } else {
15907 assert!(!runner.state.gate_ran);
15908 }
15909 assert_eq!(
15910 runner.state.status,
15911 if e2e_ok && gate_ok {
15912 RunStatus::Ready
15913 } else {
15914 RunStatus::Blocked
15915 }
15916 );
15917 if !e2e_ok || !gate_ok {
15918 let cause = runner.state.verification_blocking_cause().unwrap();
15919 assert!(cause.contains(&head));
15920 assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
15921 runner.state.event("land", "rebased successfully");
15923 let notice = crate::notices::run_ended(&runner.state).unwrap();
15924 assert!(!format!("{notice:?}").contains("rebased successfully"));
15925 assert!(runner.state.merge.is_none());
15926 }
15927 let before = std::fs::read_to_string(&phases).unwrap();
15928 runner.state.save().unwrap();
15929 runner.state = RunState::load(&runner.state.id).unwrap();
15930 runner.review_loop().await.unwrap();
15931 runner.gate().await.unwrap();
15932 assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
15933 assert_eq!(runner.state.verification_history.len(), 1);
15934 let history = &runner.state.verification_history[0];
15935 assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
15936 assert!(history.archived_at.is_some());
15937 assert_eq!(history.fixes_spent, Some(1));
15938 }
15939 }
15940
15941 #[cfg(unix)]
15942 #[tokio::test]
15943 async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
15944 let (tmp, mut runner, head) = resumed_gate_fixture().await;
15945 let marker = tmp.path().join("gate-ran");
15946 runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
15947 runner.review_loop().await.unwrap();
15948 runner.gate().await.unwrap();
15949 assert_eq!(runner.state.status, RunStatus::Blocked);
15950 assert!(!marker.exists());
15951 assert!(runner.state.verification_history.is_empty());
15952 runner.state.gate_head = None;
15953 runner.gate().await.unwrap();
15954 assert!(marker.exists());
15955 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
15956 assert_eq!(runner.state.verification_history.len(), 1);
15957 assert_eq!(runner.state.gate_fixes.len(), 1);
15958 }
15959
15960 #[cfg(unix)]
15961 #[tokio::test]
15962 async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
15963 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
15964 runner.sync_to_base().await.unwrap();
15965 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15966 runner.state.reconcile_verification_head(&head);
15967 for outcomes in [
15969 Vec::new(),
15970 vec![CommandOutcome {
15971 command: "true".into(),
15972 code: None,
15973 output_tail: "cache unavailable".into(),
15974 duration_ms: 0,
15975 resource_blocked: true,
15976 }],
15977 ] {
15978 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
15979 head: head.clone(),
15980 passed: false,
15981 outcomes,
15982 });
15983 runner.state.gate_ran = false;
15984 runner.state.gate.clear();
15985 runner.state.save().unwrap();
15986 runner.state = RunState::load(&runner.state.id).unwrap();
15987 runner.merge().await.unwrap();
15988 assert!(runner.state.merge.is_none());
15989 runner.review_loop().await.unwrap();
15990 runner.gate().await.unwrap();
15991 assert!(runner.state.head_e2e.as_ref().unwrap().passed);
15992 assert!(runner.state.gate_allows_progress());
15993 assert_eq!(runner.state.gate_fixes.len(), 1);
15994 assert!(runner.state.merge.is_none());
15995 }
15996 git::git(
15998 &runner.state.repo,
15999 &["commit", "--allow-empty", "-m", "external update"],
16000 )
16001 .await
16002 .unwrap();
16003 runner.merge().await.unwrap();
16004 assert!(runner.state.merge.is_none());
16005 assert_eq!(runner.state.status, RunStatus::Blocked);
16006 }
16007
16008 #[cfg(unix)]
16009 #[tokio::test]
16010 async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
16011 for waiting_on_gate in [false, true] {
16012 let (tmp, mut runner, _old) = resumed_gate_fixture().await;
16013 runner.sync_to_base().await.unwrap();
16014 let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16015 runner.state.reconcile_verification_head(&head);
16016 if waiting_on_gate {
16017 runner.state.head_e2e = Some(crate::run::GateRepairE2e {
16018 head: head.clone(),
16019 passed: true,
16020 outcomes: Vec::new(),
16021 });
16022 }
16023 let cache = tmp.path().join("cache");
16024 runner.state.config.verify.e2e =
16025 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
16026 runner.state.config.verify.gate = vec!["true".into()];
16027 runner.state.config.graph.timeout_resource_wait = 1;
16028 let owner =
16029 crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
16030 let guard =
16031 match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
16032 crate::cache::AcquireOutcome::Acquired(g) => g,
16033 crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
16034 };
16035 runner.review_loop().await.unwrap();
16036 runner.gate().await.unwrap();
16037 assert!(!runner.state.gate_ran);
16038 assert_eq!(runner.state.gate_fixes.len(), 1);
16039 assert!(runner.state.resource_block.is_some());
16040 runner.merge().await.unwrap();
16041 assert!(runner.state.merge.is_none());
16042 drop(guard);
16043 runner.state = RunState::load(&runner.state.id).unwrap();
16044 runner.state.resource_block = None;
16046 runner.review_loop().await.unwrap();
16047 runner.gate().await.unwrap();
16048 assert!(runner.state.gate_allows_progress());
16049 assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
16050 assert_eq!(runner.state.gate_fixes.len(), 1);
16051 }
16052 }
16053
16054 #[cfg(unix)]
16055 #[tokio::test]
16056 async fn resumed_gate_pre_gate_head_change_and_review_holds() {
16057 let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
16058 runner.sync_to_base().await.unwrap();
16059 let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16060 runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
16061 runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
16063 runner.review_loop().await.unwrap();
16064 runner.gate().await.unwrap();
16065 let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16066 assert_ne!(before, after);
16067 assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
16068 assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
16069 assert!(!runner.state.gate_allows_progress());
16070 runner.merge().await.unwrap();
16071 assert!(runner.state.merge.is_none());
16072 assert!(
16073 runner
16074 .state
16075 .verification_history
16076 .iter()
16077 .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
16078 );
16079
16080 runner.state.reviews[0].clean = false;
16082 runner.state.reviews[0].answered = 0;
16083 runner.state.config.verify.e2e = vec!["true".into()];
16084 runner.review_loop().await.unwrap();
16085 runner.gate().await.unwrap();
16086 runner.merge().await.unwrap();
16087 assert_eq!(runner.state.status, RunStatus::Blocked);
16088 assert!(runner.state.merge.is_none());
16089 assert!(
16090 runner
16091 .state
16092 .verification_blocking_cause()
16093 .unwrap()
16094 .contains("review blocked")
16095 );
16096 }
16097
16098 #[cfg(unix)]
16099 #[tokio::test]
16100 async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
16101 let (_tmp, mut runner, old) = resumed_gate_fixture().await;
16102 runner.state.gate_head = None;
16103 runner.state.config.verify.gate =
16104 vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
16105 runner.review_loop().await.unwrap();
16106 runner.gate().await.unwrap();
16107 assert_ne!(
16108 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16109 old
16110 );
16111 assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
16112 assert!(!runner.state.gate_allows_progress());
16113 assert_eq!(runner.state.status, RunStatus::Blocked);
16114 runner.merge().await.unwrap();
16115 assert!(runner.state.merge.is_none());
16116 }
16117
16118 #[cfg(unix)]
16119 #[tokio::test]
16120 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
16121 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16122 let mut old_review = review_round(true, 0, 1, 1, false, true);
16123 old_review.head = head.clone();
16124 old_review.verified_head = Some(head.clone());
16125 runner.state.reviews = vec![old_review];
16126 runner.state.gate = vec![CommandOutcome {
16127 command: "old gate".into(),
16128 code: None,
16129 output_tail: "timed out after 2400s".into(),
16130 duration_ms: 2_400_000,
16131 resource_blocked: false,
16132 }];
16133 runner.state.gate_ran = true;
16134 runner.state.gate_head = Some(head.clone());
16135 runner.state.land_approval = Some(crate::run::LandApproval {
16136 question: "old-head-approval".into(),
16137 head: head.clone(),
16138 });
16139 runner.state.authorize_base_sync(&task).unwrap();
16140 runner.sync_to_base().await.unwrap();
16141 assert!(!runner.state.base_sync_blocked());
16142 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
16143 assert_ne!(head, fresh);
16144 assert!(runner.state.reviews.is_empty());
16145 assert!(!runner.state.gate_ran);
16146 let saved = &runner.state.base_sync_recovery.authorizations[0];
16147 assert_eq!(saved.reviews[0].head, head);
16148 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
16149 assert!(saved.gate_ran);
16150 assert!(!saved.gate[0].ok());
16151 runner.review_loop().await.unwrap();
16152 runner.gate().await.unwrap();
16153 assert_eq!(runner.state.reviews[0].head, fresh);
16154 assert_eq!(
16155 runner.state.reviews[0].verified_head.as_deref(),
16156 Some(fresh.as_str())
16157 );
16158 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
16159 assert!(runner.state.gate_ran);
16160 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16161 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
16162 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
16163 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16164 assert_eq!(
16165 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
16166 "retained change"
16167 );
16168 }
16169
16170 #[cfg(unix)]
16171 #[tokio::test]
16172 async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
16173 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16174 let queue = crate::queue::Queue::open();
16175 let mut owner = queue.get(&task).unwrap();
16176 owner.release_operator("test-resume", None);
16177 queue.put(&mut owner).unwrap();
16178 let generation = owner.resumption.as_ref().unwrap().generation;
16179 runner.state.task_instruction = Some(owner.instruction.clone());
16180 runner.state.driver_exited = true;
16181 runner.state.save().unwrap();
16182 let home = crate::run::home();
16183 let _claim = queue.claim(&task).unwrap();
16184 owner.resumption.as_mut().unwrap().at =
16185 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
16186 queue.put(&mut owner).unwrap();
16187 assert!(
16188 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16189 .await
16190 .is_err(),
16191 "an approval from before the stop grants nothing"
16192 );
16193 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16194 queue.put(&mut owner).unwrap();
16195 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16196 .await
16197 .unwrap();
16198 assert!(
16199 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16200 .await
16201 .is_err()
16202 );
16203 runner.state = RunState::load(&runner.state.id).unwrap();
16204 assert_eq!(
16205 runner.state.base_sync_recovery.authorizations[0].generation,
16206 Some(generation)
16207 );
16208 runner.sync_to_base().await.unwrap();
16209 let repo = runner.state.repo.clone();
16210 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16211 git::git(&repo, &["checkout", "main"]).await.unwrap();
16212 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
16213 git::git(&repo, &["add", "."]).await.unwrap();
16214 git::git(&repo, &["commit", "-m", "move base again"])
16215 .await
16216 .unwrap();
16217 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16218 git::git(&repo, &["checkout", "fix/retained"])
16219 .await
16220 .unwrap();
16221 runner.finish_after_tally().await.unwrap();
16222 assert_eq!(runner.state.status, RunStatus::Blocked);
16223 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16224 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16225 assert!(!runner.state.base_recovery_pending());
16226 assert!(!crate::handover::can_resume(&runner.state));
16227 assert!(!runner.state.gate_ran);
16228 owner.resumption.as_mut().unwrap().at = Timestamp::now();
16230 queue.put(&mut owner).unwrap();
16231 assert!(
16232 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16233 .await
16234 .is_err()
16235 );
16236 runner.state = RunState::load(&runner.state.id).unwrap();
16237 runner.sync_to_base().await.unwrap();
16238 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16239 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
16240 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16241 }
16242
16243 #[cfg(unix)]
16244 #[tokio::test]
16245 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
16246 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
16247 runner.state.base_sync = None;
16248 runner.state.status = RunStatus::Blocked;
16249 let mut second = runner.state.candidates[0].clone();
16250 second.label = 'B';
16251 runner.state.candidates.push(second);
16252 assert!(runner.state.votes.is_empty());
16253 assert!(runner.state.deliberation.is_empty());
16254 runner.deliberate().await.unwrap();
16255 runner.vote().await.unwrap();
16256 assert_eq!(runner.state.status, RunStatus::Blocked);
16257 assert!(runner.state.votes.is_empty());
16258 assert!(runner.state.deliberation.is_empty());
16259 }
16260
16261 fn runner_at(status: RunStatus) -> Runner {
16264 let mut state = RunState::new(
16265 PathBuf::from("/nonexistent/repo"),
16266 "main".to_owned(),
16267 "deadbeef".to_owned(),
16268 "task".to_owned(),
16269 Config::default(),
16270 );
16271 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
16274 state.id = format!(
16275 "20000101-000000-{:04x}",
16276 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
16277 );
16278 state.status = status;
16279 Runner {
16280 state,
16281 roles: ResolvedRoles {
16282 implementers: Vec::new(),
16283 judges: Vec::new(),
16284 reviewers: Vec::new(),
16285 fixer: None,
16286 conductor: conductor(),
16287 implementer_roster: Vec::new(),
16288 judge_roster: Vec::new(),
16289 reviewer_roster: Vec::new(),
16290 },
16291 sem: Arc::new(Semaphore::new(1)),
16292 pause: Pause::new(),
16293 interrupt: Pause::new(),
16294 }
16295 }
16296
16297 #[tokio::test]
16298 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
16299 crate::run::pin_test_home();
16300 let mut runner = runner_at(RunStatus::Gating);
16301 runner.state.save().unwrap();
16302 let id = runner.state.id.clone();
16303 let passes = runner.state.passes.len();
16304 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16305 let mut claiming = Box::pin(runner.claim_driver());
16306 std::future::poll_fn(|cx| {
16307 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
16308 std::task::Poll::Ready(())
16309 })
16310 .await;
16311 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
16312 drop(owner);
16313 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
16314 .await
16315 .unwrap()
16316 .unwrap();
16317 assert_eq!(execution.runner.state.passes.len(), passes + 1);
16318 assert!(execution.runner.state.resource_block.is_none());
16319 assert_eq!(
16320 execution.runner.state.liveness(false),
16321 crate::run::Liveness::Live
16322 );
16323 drop(execution);
16324 assert!(RunState::load(&id).unwrap().driver_exited);
16325 }
16326
16327 #[tokio::test]
16328 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
16329 crate::run::pin_test_home();
16330 let mut runner = runner_at(RunStatus::Gating);
16331 runner.state.save().unwrap();
16332 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
16335 assert!(runner.claim_driver().await.is_err());
16336 assert!(runner.state.driver_exited);
16337 assert_eq!(runner.state.driver_pid, None);
16338 let disk = RunState::load(&runner.state.id).unwrap();
16339 assert!(disk.driver_exited);
16340 assert_eq!(disk.driver_pid, None);
16341 assert_eq!(disk.status, RunStatus::Gating);
16342 }
16343
16344 #[tokio::test]
16345 async fn an_early_graph_error_releases_its_driver() {
16346 crate::run::pin_test_home();
16347 let mut runner = runner_at(RunStatus::Blocked);
16348 runner.state.config.graph.selected_implementer = Some("selected".into());
16349 runner.state.save().unwrap();
16350 assert!(runner.execute().await.is_err());
16351 let disk = RunState::load(&runner.state.id).unwrap();
16352 assert!(disk.driver_exited);
16353 assert_eq!(disk.driver_pid, None);
16354 assert_eq!(disk.status, RunStatus::Blocked);
16355 }
16356
16357 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
16358 crate::run::MergeRepair {
16359 question: "repair-question".into(),
16360 head: "old-head".into(),
16361 pr: "https://github.com/o/r/pull/42".into(),
16362 reviews: vec![review_round(false, 1, 1, 1, false, true)],
16363 gate: vec![outcome(Some(0), false)],
16364 gate_head: Some("old-head".into()),
16365 gate_fixes: Vec::new(),
16366 pre_gate: Vec::new(),
16367 phase,
16368 failure: None,
16369 }
16370 }
16371
16372 #[tokio::test]
16373 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
16374 crate::run::pin_test_home();
16375 let mut runner = runner_at(RunStatus::Landing);
16376 runner.state.merge_repairs.push(merge_repair_request(
16377 crate::run::MergeRepairPhase::Requested,
16378 ));
16379 runner.state.save().unwrap();
16380 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
16381 assert!(
16382 runner
16383 .execute()
16384 .await
16385 .unwrap_err()
16386 .to_string()
16387 .contains("already running")
16388 );
16389 assert_eq!(
16390 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16391 crate::run::MergeRepairPhase::Requested
16392 );
16393 drop(claim);
16394 let mut disk = runner.state.clone();
16395 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
16396 disk.save().unwrap();
16397 assert!(
16398 runner
16399 .execute()
16400 .await
16401 .unwrap_err()
16402 .to_string()
16403 .contains("decision changed")
16404 );
16405 assert_eq!(
16406 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
16407 crate::run::MergeRepairPhase::Complete
16408 );
16409 }
16410
16411 #[test]
16412 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
16413 crate::run::pin_test_home();
16414 let mut runner = runner_at(RunStatus::Landing);
16415 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
16416 runner.state.reviews = request.reviews.clone();
16417 runner.state.gate = request.gate.clone();
16418 runner.state.gate_ran = true;
16419 runner.state.land_approval = Some(crate::run::LandApproval {
16420 question: "old".into(),
16421 head: "old-head".into(),
16422 });
16423 runner.state.merge_repairs.push(request);
16424 runner.begin_merge_repair_review(0).unwrap();
16425 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
16426 assert!(!runner.state.gate_status().ok());
16427 assert!(runner.state.land_approval.is_none());
16428 let disk = RunState::load(&runner.state.id).unwrap();
16429 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
16430 assert!(disk.merge_repairs[0].gate[0].ok());
16431 assert_eq!(
16432 disk.merge_repairs[0].phase,
16433 crate::run::MergeRepairPhase::Reviewing
16434 );
16435 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
16436 assert!(disk.reviews.is_empty());
16437 }
16438
16439 #[tokio::test]
16440 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
16441 crate::run::pin_test_home();
16442 for phase in [
16443 crate::run::MergeRepairPhase::Fixing,
16444 crate::run::MergeRepairPhase::Requested,
16445 ] {
16446 let mut runner = runner_at(RunStatus::Landing);
16447 runner.state.config.graph.land_rounds = 0;
16448 runner.state.merge_repairs.push(merge_repair_request(phase));
16449 runner.state.save().unwrap();
16450 assert!(!runner.repair_merge_review().await.unwrap());
16451 assert_eq!(runner.state.status, RunStatus::Blocked);
16452 assert_eq!(
16453 runner.state.merge_repairs[0].phase,
16454 crate::run::MergeRepairPhase::Failed
16455 );
16456 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
16457 assert!(
16458 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
16459 "interrupted"
16460 } else {
16461 "budget exhausted"
16462 })
16463 );
16464 assert!(runner.state.merge.is_none());
16465 }
16466 }
16467
16468 #[test]
16472 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
16473 crate::run::pin_test_home();
16474 let mut runner = runner_at(RunStatus::Implementing);
16475 let interrupt = Pause::new();
16476 runner.watch_interrupt(interrupt.clone());
16477
16478 interrupt.park_because("task a1b2 asked to run first");
16479
16480 assert!(runner.park_here().expect("park_here"));
16481 assert!(runner.state.parked);
16482 let last = runner.state.events.last().expect("a park event");
16483 assert_eq!(last.node, "park");
16484 assert!(
16485 last.message.contains("task a1b2 asked to run first"),
16486 "expected the interrupt reason in {:?}",
16487 last.message
16488 );
16489 }
16490
16491 #[test]
16494 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
16495 crate::run::pin_test_home();
16496 let mut runner = runner_at(RunStatus::Implementing);
16497 runner.state.begin_pass();
16498 let shutdown = Pause::new();
16499 runner.on_pause(shutdown.clone());
16500 let upgrade = ParkCause::Upgrade {
16501 from: "0.121.0".to_owned(),
16502 to: Some("0.121.1".to_owned()),
16503 };
16504 shutdown.park_for(upgrade.clone());
16505 shutdown.park_for(ParkCause::Operator);
16506
16507 assert!(runner.park_here().expect("park_here"));
16508 assert_eq!(
16509 runner.state.passes[0].end,
16510 Some(crate::run::PassEnd::Parked { cause: upgrade })
16511 );
16512 let last = runner.state.events.last().expect("a park event");
16513 assert!(last.message.contains("0.121.0"), "{}", last.message);
16514 }
16515
16516 #[tokio::test]
16517 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
16518 crate::run::pin_test_home();
16519 let mut runner = runner_at(RunStatus::Reviewing);
16520 runner.state.save().unwrap();
16521 let mut execution = runner.claim_driver().await.expect("first execution");
16522 assert_eq!(execution.runner.state.passes.len(), 1);
16523 execution
16524 .driver
16525 .finish(&mut execution.runner.state)
16526 .await
16527 .unwrap();
16528 }
16529
16530 #[tokio::test]
16531 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
16532 crate::run::pin_test_home();
16533 let mut runner = runner_at(RunStatus::Reviewing);
16534 runner.state.config.agents = vec![conductor()];
16535 runner.state.config.graph.timeout_resource_wait = 0;
16536 runner.state.save().unwrap();
16537 let id = runner.state.id.clone();
16538 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
16539 assert!(runner.claim_driver().await.is_err());
16540 drop(owner);
16541 drop(runner);
16542 let mut resumed = Runner::resume(&id).unwrap();
16543 assert_eq!(resumed.state.driver_claimed, Some(false));
16544 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
16545 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
16546 let mut execution = resumed
16547 .claim_driver()
16548 .await
16549 .expect("retry initial claim after reload");
16550 assert_eq!(execution.runner.state.driver_claimed, Some(true));
16551 assert_eq!(execution.runner.state.passes.len(), 1);
16552 execution
16553 .driver
16554 .finish(&mut execution.runner.state)
16555 .await
16556 .unwrap();
16557 }
16558
16559 #[tokio::test]
16560 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
16561 crate::run::pin_test_home();
16562 let mut runner = runner_at(RunStatus::Reviewing);
16563 runner.state.config.agents = vec![conductor()];
16564 runner.state.save().unwrap();
16565 let id = runner.state.id.clone();
16566 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
16567 let mut resumed = Runner::resume(&id).unwrap();
16568 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
16569 assert!(resumed.claim_driver().await.is_err());
16570 assert!(RunState::load(&id).unwrap().passes.is_empty());
16571 drop(lease);
16572 let mut execution = resumed.claim_driver().await.unwrap();
16573 execution
16574 .driver
16575 .finish(&mut execution.runner.state)
16576 .await
16577 .unwrap();
16578 }
16579
16580 #[tokio::test]
16581 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
16582 crate::run::pin_test_home();
16583 let mut runner = runner_at(RunStatus::Reviewing);
16584 let state = &mut runner.state;
16585 state.config.agents = vec![conductor()];
16586 state.driver_claimed = None;
16587 state.driver_pid = Some(std::process::id());
16588 state.driver_started_at = None;
16589 state.begin_pass();
16590 state.save().unwrap();
16591 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
16592 std::fs::write(
16593 state.dir().join("driver.json"),
16594 serde_json::to_vec(&serde_json::json!({
16595 "pid": state.driver_pid, "started_at": null,
16596 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
16597 }))
16598 .unwrap(),
16599 )
16600 .unwrap();
16601 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
16602 drop(lease);
16603 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
16604 let mut resumed = Runner::resume(&state.id).unwrap();
16605 assert!(!resumed.state.driver_exited);
16606 let mut execution = resumed
16607 .claim_driver()
16608 .await
16609 .expect("lease proves previous execution ended");
16610 assert_eq!(execution.runner.state.passes.len(), 2);
16611 execution
16612 .driver
16613 .finish(&mut execution.runner.state)
16614 .await
16615 .unwrap();
16616 }
16617
16618 #[tokio::test]
16619 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
16620 crate::run::pin_test_home();
16621 let mut runner = runner_at(RunStatus::Reviewing);
16622 let mut json = serde_json::to_value(&runner.state).unwrap();
16623 json.as_object_mut().unwrap().remove("driver_claimed");
16624 runner.state = serde_json::from_value(json).unwrap();
16625 assert_eq!(runner.state.driver_claimed, None);
16626 runner.state.save().unwrap();
16627 let mut execution = runner
16628 .claim_driver()
16629 .await
16630 .expect("the execution lease, not a missing marker, excludes a second driver");
16631 assert_eq!(execution.runner.state.passes.len(), 1);
16632 execution
16633 .driver
16634 .finish(&mut execution.runner.state)
16635 .await
16636 .unwrap();
16637 }
16638
16639 #[tokio::test]
16640 async fn driver_admission_refuses_a_proven_live_driver() {
16641 crate::run::pin_test_home();
16642 let mut runner = runner_at(RunStatus::Reviewing);
16643 let pid = std::process::id();
16644 runner.state.driver_pid = Some(pid);
16645 runner.state.driver_started_at = crate::proc::process_started_at(pid);
16646 assert!(runner.state.driver_started_at.is_some());
16647 runner.state.save().unwrap();
16648 let error = runner
16649 .claim_driver()
16650 .await
16651 .err()
16652 .expect("a live driver must refuse");
16653 assert!(error.to_string().contains("live driver"), "{error:#}");
16654 let saved = RunState::load(&runner.state.id).unwrap();
16655 assert_eq!(saved.driver_pid, Some(pid));
16656 assert_eq!(saved.status, RunStatus::Reviewing);
16657 assert!(saved.passes.is_empty());
16658 }
16659
16660 #[test]
16661 fn parking_resource_halt_persists_the_pass_once() {
16662 crate::run::pin_test_home();
16663 let mut runner = runner_at(RunStatus::Reviewing);
16664 runner.state.begin_pass();
16665 runner.state.resource_block = Some(crate::run::ResourceBlock {
16666 node: "fix".into(),
16667 kind: "shared build cache".into(),
16668 detail: "cancelled".into(),
16669 at: Timestamp::now(),
16670 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16671 });
16672 let cause = ParkCause::Upgrade {
16673 from: "old".into(),
16674 to: Some("new".into()),
16675 };
16676 runner.pause.park_for(cause.clone());
16677 runner.stop_for_resource().unwrap();
16680 assert!(runner.resource_halt().unwrap());
16681 assert!(runner.park_here().unwrap());
16682 let saved = RunState::load(&runner.state.id).unwrap();
16683 assert!(saved.parked);
16684 assert_eq!(saved.status, RunStatus::Reviewing);
16685 assert!(saved.passes[0].ended_at.is_some());
16686 assert_eq!(
16687 saved.passes[0].end,
16688 Some(crate::run::PassEnd::Parked {
16689 cause: cause.clone()
16690 })
16691 );
16692 assert_eq!(
16693 saved.resource_block.unwrap().cause,
16694 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16695 );
16696 assert_eq!(
16697 saved
16698 .events
16699 .iter()
16700 .filter(|event| event.node == "park")
16701 .count(),
16702 1
16703 );
16704 }
16705
16706 #[test]
16714 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16715 crate::run::pin_test_home();
16716 let mut runner = runner_at(RunStatus::Implementing);
16717 let shutdown = Pause::new();
16718 runner.on_pause(shutdown.clone());
16719 let interrupt = Pause::new();
16720 runner.watch_interrupt(interrupt.clone());
16721
16722 assert!(!runner.park_here().expect("park_here"));
16724 assert!(!runner.state.parked);
16725
16726 interrupt.park_because("test");
16728 assert!(!shutdown.parked());
16729 assert!(runner.park_here().expect("park_here"));
16730 }
16731
16732 #[tokio::test]
16746 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16747 crate::run::pin_test_home();
16748 let mut runner = runner_at(RunStatus::Implementing);
16749 let interrupt = Pause::new();
16750 runner.watch_interrupt(interrupt.clone());
16751
16752 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16753 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16754
16755 let node = async move {
16759 started_tx.send(()).expect("send started");
16760 finish_rx.await.expect("recv finish");
16761 "node finished"
16762 };
16763
16764 let interrupter = async move {
16765 started_rx.await.expect("recv started");
16766 interrupt.park_because("higher-priority task waiting");
16768 tokio::task::yield_now().await;
16772 finish_tx.send(()).expect("send finish");
16773 };
16774
16775 let (node_result, ()) = tokio::join!(node, interrupter);
16776 assert_eq!(
16777 node_result, "node finished",
16778 "the in-flight call ran to completion"
16779 );
16780
16781 assert!(runner.park_here().expect("park_here"));
16784 assert!(runner.state.parked);
16785 }
16786
16787 #[test]
16793 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16794 crate::run::pin_test_home();
16795 let mut runner = runner_at(RunStatus::Judging);
16796 runner.state.config.agents = vec![conductor()];
16800 runner.state.candidates = vec![Candidate {
16801 index: 0,
16802 label: 'A',
16803 agent: "alpha".to_owned(),
16804 branch: "magi/x/A".to_owned(),
16805 worktree: PathBuf::from("/nonexistent/worktree"),
16806 summary: "did the thing".to_owned(),
16807 stat: "1 file changed".to_owned(),
16808 files: 1,
16809 commits: 1,
16810 empty: false,
16811 failed: None,
16812 verified_noop: None,
16813 duration_ms: 1234,
16814 folded: false,
16815 }];
16816 let run_id = runner.state.id.clone();
16817
16818 let interrupt = Pause::new();
16819 runner.watch_interrupt(interrupt.clone());
16820 interrupt.park_because("task c3d4 asked to run first");
16821 assert!(runner.park_here().expect("park_here"));
16822
16823 let resumed = Runner::resume(&run_id).expect("resume");
16824 assert_eq!(resumed.state.candidates.len(), 1);
16825 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16826 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16827 assert_eq!(resumed.state.status, runner.state.status);
16828 assert!(
16829 resumed.state.parked,
16830 "still parked until `execute` actually walks the graph again"
16831 );
16832 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16833 }
16834
16835 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16837 let mut q = ask::Question::new(
16838 run.to_owned(),
16839 "implement".to_owned(),
16840 "impl-A".to_owned(),
16841 "Which storage backend should the cache use?".to_owned(),
16842 String::new(),
16843 vec!["SQLite".to_owned(), "Redis".to_owned()],
16844 );
16845 store.put(&mut q).unwrap();
16846 q
16847 }
16848
16849 #[test]
16850 fn a_failed_runs_open_question_is_abandoned() {
16851 ask_test_home();
16852 let store = ask::Questions::open();
16853 let mut runner = runner_at(RunStatus::Failed);
16854 let run = runner.state.id.clone();
16855 let q = ask_open_question(&store, &run);
16856
16857 runner.settle_questions();
16858
16859 let back = store.get(&q.id).unwrap();
16860 assert!(
16861 !back.status.open(),
16862 "the seat that asked died with the run; nobody is left to read an answer"
16863 );
16864 assert!(
16865 back.detail.contains(&run) && back.detail.contains("failed"),
16866 "the reason names what the run became, not just that it is gone: {}",
16867 back.detail
16868 );
16869 }
16870
16871 #[test]
16872 fn a_merged_runs_open_question_is_abandoned_too() {
16873 ask_test_home();
16874 let store = ask::Questions::open();
16875 for status in [RunStatus::Merged, RunStatus::Ready] {
16878 let mut runner = runner_at(status);
16879 let run = runner.state.id.clone();
16880 let q = ask_open_question(&store, &run);
16881
16882 runner.settle_questions();
16883
16884 let back = store.get(&q.id).unwrap();
16885 assert!(
16886 !back.status.open(),
16887 "{status:?} run's question must not outlive the run"
16888 );
16889 }
16890 }
16891
16892 #[test]
16893 fn a_still_resumable_runs_open_question_is_left_alone() {
16894 ask_test_home();
16895 let store = ask::Questions::open();
16896 for status in [RunStatus::Blocked, RunStatus::Stalled] {
16902 let mut runner = runner_at(status);
16903 let run = runner.state.id.clone();
16904 let q = ask_open_question(&store, &run);
16905
16906 runner.settle_questions();
16907
16908 let back = store.get(&q.id).unwrap();
16909 assert!(
16910 back.status.open(),
16911 "{status:?} is still alive; the question must still be waiting"
16912 );
16913 }
16914 }
16915
16916 #[test]
16917 fn settle_questions_never_touches_an_already_answered_question() {
16918 ask_test_home();
16919 let store = ask::Questions::open();
16920 let mut runner = runner_at(RunStatus::Failed);
16921 let run = runner.state.id.clone();
16922 let mut q = ask_open_question(&store, &run);
16923 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16924 .unwrap();
16925 store.put(&mut q).unwrap();
16926
16927 runner.settle_questions();
16932 runner.settle_questions();
16933
16934 let back = store.get(&q.id).unwrap();
16935 assert_eq!(
16936 back.status,
16937 ask::QuestionStatus::Answered,
16938 "a real answer is a decision on record, never overwritten by a sweep"
16939 );
16940 }
16941
16942 #[tokio::test]
16953 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16954 crate::run::pin_test_home();
16955 let tmp = tempfile::tempdir().expect("tempdir");
16956 let repo = tmp.path().join("repo");
16957 std::fs::create_dir_all(&repo).unwrap();
16958 init_repo(&repo);
16959
16960 let mut config = Config::default();
16961 config.graph.worktree_root = Some(tmp.path().join("wt"));
16962
16963 let mut state = RunState::new(
16964 repo.clone(),
16965 "main".to_owned(),
16966 "deadbeef".to_owned(),
16967 "task".to_owned(),
16968 config,
16969 );
16970 let root = state.worktree_root();
16971 let wt_a = root.join("cand-A");
16972 let wt_b = root.join("cand-B");
16973 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16974 .await
16975 .expect("worktree A");
16976 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16977 .await
16978 .expect("worktree B");
16979
16980 state.candidates = vec![
16981 Candidate {
16982 index: 0,
16983 label: 'A',
16984 agent: "alpha".to_owned(),
16985 branch: "magi/x/A".to_owned(),
16986 worktree: wt_a.clone(),
16987 summary: String::new(),
16988 stat: String::new(),
16989 files: 0,
16990 commits: 0,
16991 empty: false,
16992 failed: None,
16993 verified_noop: None,
16994 duration_ms: 0,
16995 folded: false,
16996 },
16997 Candidate {
16998 index: 1,
16999 label: 'B',
17000 agent: "beta".to_owned(),
17001 branch: "magi/x/B".to_owned(),
17002 worktree: wt_b.clone(),
17003 summary: String::new(),
17004 stat: String::new(),
17005 files: 0,
17006 commits: 0,
17007 empty: false,
17008 failed: None,
17009 verified_noop: None,
17010 duration_ms: 0,
17011 folded: false,
17012 },
17013 ];
17014 state.tally = Some(Tally {
17015 first_choice: BTreeMap::from([('A', 1)]),
17016 borda: BTreeMap::new(),
17017 winner: 'A',
17018 rankings: 1,
17019 unanimous_initial: true,
17020 deliberated: false,
17021 changed_votes: 0,
17022 unanimous_final: true,
17023 tie_break: None,
17024 judges: 1,
17025 present: 1,
17026 quorum: 1,
17027 met_quorum: true,
17028 uncontested: None,
17029 });
17030 state.status = RunStatus::Ready;
17031
17032 fold_run(&mut state, false, &crate::run::home())
17033 .await
17034 .expect("fold_run");
17035
17036 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
17037 assert!(
17038 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17039 "the unmerged winner's branch survives"
17040 );
17041 assert!(
17042 !state.candidates[0].folded,
17043 "the winner is not marked folded"
17044 );
17045
17046 assert!(!wt_b.exists(), "the loser's worktree is removed");
17047 assert!(
17048 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
17049 "the loser's branch is removed"
17050 );
17051 assert!(state.candidates[1].folded, "the loser is marked folded");
17052 }
17053
17054 #[tokio::test]
17055 #[cfg(unix)]
17056 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
17057 use std::os::fd::AsRawFd as _;
17058 use std::time::Duration;
17059
17060 let tmp = tempfile::tempdir().unwrap();
17061 let repo = tmp.path().join("repo");
17062 std::fs::create_dir_all(&repo).unwrap();
17063 init_repo(&repo);
17064 let common = repo.join(".git");
17065 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
17066 let home = tmp.path().join("home");
17067 let mut config = Config::default();
17068 config.coordination = crate::config::test_coordination();
17069 config.graph.worktree_root = Some(tmp.path().join("wt"));
17070 config.graph.timeout_resource_wait = 1;
17071 let mut state = RunState::new(
17072 repo.clone(),
17073 "main".into(),
17074 "deadbeef".into(),
17075 "task".into(),
17076 config,
17077 );
17078 let root = state.worktree_root();
17079 let candidate = root.join("cand-A");
17080 let reviewer = root.join("review-1");
17081 let branch = state.branch_for('A');
17082 git::worktree_add_branch(&repo, &candidate, &branch, "main")
17083 .await
17084 .unwrap();
17085 git::worktree_add_detached(&repo, &reviewer, "main")
17086 .await
17087 .unwrap();
17088 git::acquire_worktree_config(&repo).await.unwrap();
17089 state.enabled_worktree_config = true;
17090 state.status = RunStatus::Ready;
17091 state.candidates.push(Candidate {
17092 index: 0,
17093 label: 'A',
17094 agent: "implement".into(),
17095 branch: branch.clone(),
17096 worktree: candidate.clone(),
17097 summary: String::new(),
17098 stat: String::new(),
17099 files: 0,
17100 commits: 0,
17101 empty: false,
17102 failed: None,
17103 verified_noop: None,
17104 duration_ms: 0,
17105 folded: false,
17106 });
17107 state.save_under(&home).unwrap();
17108 let config_before = std::fs::read(common.join("config")).unwrap();
17109 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
17110 .await
17111 .unwrap();
17112 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
17115 assert_eq!(
17116 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
17117 0
17118 );
17119 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17120 assert!(
17121 tokio::time::timeout(Duration::from_millis(200), &mut fold)
17122 .await
17123 .is_err()
17124 );
17125 assert!(candidate.exists());
17126 assert!(reviewer.exists());
17127 assert!(git::branch_exists(&repo, &branch).await.unwrap());
17128 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17129 assert_eq!(
17130 git::git(&repo, &["worktree", "list", "--porcelain"])
17131 .await
17132 .unwrap(),
17133 worktrees_before
17134 );
17135 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
17136 .await
17137 .unwrap()
17138 .unwrap_err();
17139 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
17140 drop(fold);
17141 assert!(candidate.exists() && reviewer.exists());
17142 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
17143 assert_eq!(
17144 state.resource_block.as_ref().unwrap().kind,
17145 "repository coordination"
17146 );
17147 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
17148 drop(external);
17149 let mut fold = Box::pin(fold_run(&mut state, true, &home));
17150 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
17151 .await
17152 .unwrap()
17153 .unwrap();
17154 drop(fold);
17155 assert!(removed.contains(&branch));
17156 assert!(!root.exists());
17157 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
17158 assert!(state.candidates[0].folded);
17159 assert!(!state.enabled_worktree_config);
17160 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
17161 let enabled = git::git_raw(
17162 &repo,
17163 &["config", "--local", "--get", "extensions.worktreeConfig"],
17164 )
17165 .await
17166 .unwrap();
17167 assert!(!enabled.ok(), "the last config reference was released");
17168 }
17169
17170 #[tokio::test]
17173 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
17174 let tmp = tempfile::tempdir().expect("tempdir");
17175 let repo = tmp.path().join("repo");
17176 std::fs::create_dir_all(&repo).unwrap();
17177 init_repo(&repo);
17178 let home = tmp.path().join("home");
17179
17180 let mut config = Config::default();
17181 config.graph.worktree_root = Some(tmp.path().join("wt"));
17182 let mut state = RunState::new(
17183 repo.clone(),
17184 "main".to_owned(),
17185 "deadbeef".to_owned(),
17186 "task".to_owned(),
17187 config,
17188 );
17189 git::git(&repo, &["branch", "magi/x/A", "main"])
17191 .await
17192 .expect("branch");
17193 state.candidates = vec![Candidate {
17194 index: 0,
17195 label: 'A',
17196 agent: "alpha".to_owned(),
17197 branch: "magi/x/A".to_owned(),
17198 worktree: state.worktree_root().join("cand-A"),
17199 summary: String::new(),
17200 stat: String::new(),
17201 files: 0,
17202 commits: 0,
17203 empty: false,
17204 failed: None,
17205 verified_noop: None,
17206 duration_ms: 0,
17207 folded: true,
17208 }];
17209 state.released_to = Some("20260901-000000-new1".to_owned());
17210 state.released_branches = vec!["magi/x/A".to_owned()];
17211
17212 fold_run(&mut state, true, &home).await.expect("fold_run");
17213 state.status = RunStatus::Merged;
17214 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
17215 state.merge = Some(crate::run::MergeOutcome {
17216 mode: MergeMode::Pr,
17217 ok: true,
17218 detail: "confirmed merged".into(),
17219 empty: false,
17220 });
17221 fold_run(&mut state, true, &home)
17222 .await
17223 .expect("merged fold");
17224
17225 assert!(
17226 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
17227 "the handed-over branch survives a fold"
17228 );
17229 }
17230
17231 #[tokio::test]
17235 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
17236 let tmp = tempfile::tempdir().expect("tempdir");
17237 let repo = tmp.path().join("repo");
17238 std::fs::create_dir_all(&repo).unwrap();
17239 init_repo(&repo);
17240 let run = |args: &[&str]| {
17241 let out = std::process::Command::new("git")
17242 .quiet()
17243 .args(args)
17244 .current_dir(&repo)
17245 .output()
17246 .expect("spawn git");
17247 assert!(out.status.success(), "git {args:?}");
17248 };
17249 run(&["branch", "magi/x/A"]);
17250 run(&["checkout", "-q", "-b", "magi/x/B"]);
17251 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
17252 run(&["add", "-A"]);
17253 run(&["commit", "-q", "-m", "work"]);
17254 run(&["checkout", "-q", "main"]);
17255
17256 let probe = RunState::new(
17260 repo.clone(),
17261 "main".to_owned(),
17262 "deadbeef".to_owned(),
17263 "task".to_owned(),
17264 Config::default(),
17265 );
17266 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
17267 let bare = tmp.path().join("origin.git");
17268 let out = std::process::Command::new("git")
17269 .quiet()
17270 .args(["init", "-q", "--bare"])
17271 .arg(&bare)
17272 .output()
17273 .expect("spawn git");
17274 assert!(out.status.success(), "git init --bare");
17275 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
17276 run(&["push", "-q", "origin", "main"]);
17277
17278 let mut state = RunState::new(
17279 repo.clone(),
17280 "main".to_owned(),
17281 "deadbeef".to_owned(),
17282 "task".to_owned(),
17283 Config::default(),
17284 );
17285 state.seeds = vec![refs::Seed {
17286 token: "magi/27b2/A".to_owned(),
17287 kind: refs::SeedKind::Unresolved,
17288 sha: String::new(),
17289 branch: true,
17290 detail: "no branch or commit named magi/27b2/A".to_owned(),
17291 }];
17292
17293 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
17294 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
17295 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
17296 let detail = empty_candidate_detail(&state, "main");
17297 assert!(detail.starts_with("empty candidate"), "{detail}");
17298 assert!(!detail.contains("magi/27b2/A"), "{detail}");
17299 assert!(detail.contains("--start-from"), "{detail}");
17300 }
17301
17302 #[tokio::test]
17311 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
17312 let tmp = tempfile::tempdir().expect("tempdir");
17313 let repo = tmp.path().join("repo");
17314 std::fs::create_dir_all(&repo).unwrap();
17315 init_repo(&repo);
17316
17317 let mut config = Config::default();
17318 config.merge.mode = MergeMode::Local;
17319
17320 let mut state = RunState::new(
17321 repo.clone(),
17322 "main".to_owned(),
17323 "deadbeef".to_owned(),
17324 "task".to_owned(),
17325 config,
17326 );
17327 state.candidates = vec![Candidate {
17328 index: 0,
17329 label: 'A',
17330 agent: "alpha".to_owned(),
17331 branch: "does-not-exist".to_owned(),
17332 worktree: repo.clone(),
17333 summary: String::new(),
17334 stat: String::new(),
17335 files: 0,
17336 commits: 0,
17337 empty: false,
17338 failed: None,
17339 verified_noop: None,
17340 duration_ms: 0,
17341 folded: false,
17342 }];
17343 state.tally = Some(Tally {
17344 first_choice: BTreeMap::from([('A', 1)]),
17345 borda: BTreeMap::new(),
17346 winner: 'A',
17347 rankings: 1,
17348 unanimous_initial: true,
17349 deliberated: false,
17350 changed_votes: 0,
17351 unanimous_final: true,
17352 tie_break: None,
17353 judges: 0,
17354 present: 0,
17355 quorum: 0,
17356 met_quorum: true,
17357 uncontested: Some("only candidate A produced a change".to_owned()),
17358 });
17359 state.reviews = vec![ReviewRound {
17360 round: 1,
17361 head: "deadbeef".to_owned(),
17362 verified_head: None,
17363 verified_at: None,
17364 reviews: Vec::new(),
17365 e2e: Vec::new(),
17366 fix: None,
17367 blocking: 0,
17368 answered: 0,
17369 expected: 0,
17370 clean: true,
17371 verify_retried: false,
17372 e2e_deferred: false,
17373 e2e_defer_reason: None,
17374 progressed: false,
17375 vote_split: false,
17376 reconsideration: Vec::new(),
17377 verdict: None,
17378 }];
17379 state.gate = vec![CommandOutcome {
17380 command: "test".to_owned(),
17381 code: Some(0),
17382 output_tail: String::new(),
17383 duration_ms: 0,
17384 resource_blocked: false,
17385 }];
17386 state.gate_ran = true;
17387 state.status = RunStatus::Ready;
17392 state.merge = Some(MergeOutcome {
17393 mode: MergeMode::Local,
17394 ok: false,
17395 detail: "already concluded".to_owned(),
17396 empty: false,
17397 });
17398
17399 let mut runner = Runner {
17400 state,
17401 roles: ResolvedRoles {
17402 implementers: Vec::new(),
17403 judges: Vec::new(),
17404 reviewers: Vec::new(),
17405 fixer: None,
17406 conductor: conductor(),
17407 implementer_roster: Vec::new(),
17408 judge_roster: Vec::new(),
17409 reviewer_roster: Vec::new(),
17410 },
17411 sem: Arc::new(Semaphore::new(1)),
17412 pause: Pause::new(),
17413 interrupt: Pause::new(),
17414 };
17415
17416 runner.merge().await.expect("merge");
17417
17418 assert_eq!(
17419 runner.state.status,
17420 RunStatus::Ready,
17421 "a concluded run's status must not change on reentry"
17422 );
17423 assert_eq!(
17424 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17425 Some("already concluded"),
17426 "merge must not run again once the node already recorded an outcome"
17427 );
17428 }
17429
17430 #[tokio::test]
17439 async fn merge_refuses_a_gate_that_has_not_actually_run() {
17440 let tmp = tempfile::tempdir().expect("tempdir");
17441 let repo = tmp.path().join("repo");
17442 std::fs::create_dir_all(&repo).unwrap();
17443 init_repo(&repo);
17444
17445 let mut config = Config::default();
17446 config.merge.mode = MergeMode::Local;
17447
17448 let mut state = RunState::new(
17449 repo.clone(),
17450 "main".to_owned(),
17451 "deadbeef".to_owned(),
17452 "task".to_owned(),
17453 config,
17454 );
17455 state.candidates = vec![Candidate {
17456 index: 0,
17457 label: 'A',
17458 agent: "alpha".to_owned(),
17459 branch: "does-not-exist".to_owned(),
17460 worktree: repo.clone(),
17461 summary: String::new(),
17462 stat: String::new(),
17463 files: 0,
17464 commits: 0,
17465 empty: false,
17466 failed: None,
17467 verified_noop: None,
17468 duration_ms: 0,
17469 folded: false,
17470 }];
17471 state.tally = Some(Tally {
17472 first_choice: BTreeMap::from([('A', 1)]),
17473 borda: BTreeMap::new(),
17474 winner: 'A',
17475 rankings: 1,
17476 unanimous_initial: true,
17477 deliberated: false,
17478 changed_votes: 0,
17479 unanimous_final: true,
17480 tie_break: None,
17481 judges: 0,
17482 present: 0,
17483 quorum: 0,
17484 met_quorum: true,
17485 uncontested: Some("only candidate A produced a change".to_owned()),
17486 });
17487 state.reviews = vec![ReviewRound {
17488 round: 1,
17489 head: "deadbeef".to_owned(),
17490 verified_head: None,
17491 verified_at: None,
17492 reviews: Vec::new(),
17493 e2e: Vec::new(),
17494 fix: None,
17495 blocking: 0,
17496 answered: 0,
17497 expected: 0,
17498 clean: true,
17499 verify_retried: false,
17500 e2e_deferred: false,
17501 e2e_defer_reason: None,
17502 progressed: false,
17503 vote_split: false,
17504 reconsideration: Vec::new(),
17505 verdict: None,
17506 }];
17507 state.gate = Vec::new();
17509 state.gate_ran = false;
17510 state.status = RunStatus::Gating;
17511
17512 let mut runner = Runner {
17513 state,
17514 roles: ResolvedRoles {
17515 implementers: Vec::new(),
17516 judges: Vec::new(),
17517 reviewers: Vec::new(),
17518 fixer: None,
17519 conductor: conductor(),
17520 implementer_roster: Vec::new(),
17521 judge_roster: Vec::new(),
17522 reviewer_roster: Vec::new(),
17523 },
17524 sem: Arc::new(Semaphore::new(1)),
17525 pause: Pause::new(),
17526 interrupt: Pause::new(),
17527 };
17528
17529 runner.merge().await.expect("merge");
17530
17531 assert!(
17532 runner.state.merge.is_none(),
17533 "an empty gate must never be read as a passing one: {:?}",
17534 runner.state.merge
17535 );
17536 }
17537
17538 #[tokio::test]
17545 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
17546 let tmp = tempfile::tempdir().expect("tempdir");
17547 let repo = tmp.path().join("repo");
17548 std::fs::create_dir_all(&repo).unwrap();
17549 init_repo(&repo);
17550
17551 let config = Config::default();
17553
17554 let mut state = RunState::new(
17555 repo.clone(),
17556 "main".to_owned(),
17557 "deadbeef".to_owned(),
17558 "task".to_owned(),
17559 config,
17560 );
17561 state.candidates = vec![Candidate {
17562 index: 0,
17563 label: 'A',
17564 agent: "alpha".to_owned(),
17565 branch: "does-not-exist".to_owned(),
17566 worktree: repo.clone(),
17567 summary: String::new(),
17568 stat: String::new(),
17569 files: 0,
17570 commits: 0,
17571 empty: false,
17572 failed: None,
17573 verified_noop: None,
17574 duration_ms: 0,
17575 folded: false,
17576 }];
17577 state.tally = Some(Tally {
17578 first_choice: BTreeMap::from([('A', 1)]),
17579 borda: BTreeMap::new(),
17580 winner: 'A',
17581 rankings: 1,
17582 unanimous_initial: true,
17583 deliberated: false,
17584 changed_votes: 0,
17585 unanimous_final: true,
17586 tie_break: None,
17587 judges: 0,
17588 present: 0,
17589 quorum: 0,
17590 met_quorum: true,
17591 uncontested: Some("only candidate A produced a change".to_owned()),
17592 });
17593 state.reviews = vec![ReviewRound {
17594 round: 1,
17595 head: "deadbeef".to_owned(),
17596 verified_head: None,
17597 verified_at: None,
17598 reviews: Vec::new(),
17599 e2e: Vec::new(),
17600 fix: None,
17601 blocking: 0,
17602 answered: 0,
17603 expected: 0,
17604 clean: true,
17605 verify_retried: false,
17606 e2e_deferred: false,
17607 e2e_defer_reason: None,
17608 progressed: false,
17609 vote_split: false,
17610 reconsideration: Vec::new(),
17611 verdict: None,
17612 }];
17613
17614 let mut runner = Runner {
17615 state,
17616 roles: ResolvedRoles {
17617 implementers: Vec::new(),
17618 judges: Vec::new(),
17619 reviewers: Vec::new(),
17620 fixer: None,
17621 conductor: conductor(),
17622 implementer_roster: Vec::new(),
17623 judge_roster: Vec::new(),
17624 reviewer_roster: Vec::new(),
17625 },
17626 sem: Arc::new(Semaphore::new(1)),
17627 pause: Pause::new(),
17628 interrupt: Pause::new(),
17629 };
17630
17631 runner.gate().await.expect("gate");
17632 assert!(
17633 runner.state.gate_ran,
17634 "zero configured commands is still a real attempt, not an unrun gate"
17635 );
17636 assert!(runner.state.gate.is_empty());
17637 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
17638 assert_ne!(
17639 runner.state.status,
17640 RunStatus::Blocked,
17641 "a gate with nothing to check must not read as failed"
17642 );
17643
17644 runner.merge().await.expect("merge");
17645 assert_eq!(
17646 runner.state.status,
17647 RunStatus::Ready,
17648 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
17649 );
17650 }
17651
17652 #[tokio::test]
17663 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17664 crate::run::pin_test_home();
17665 let home = crate::run::home();
17666
17667 let tmp = tempfile::tempdir().expect("tempdir");
17668 let repo = tmp.path().join("repo");
17669 std::fs::create_dir_all(&repo).unwrap();
17670 init_repo(&repo);
17671 let cache_dir = tmp.path().join("target");
17674
17675 let mut config = Config::default();
17676 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17677 config.graph.timeout_verify = Some(2);
17680 config.graph.timeout_resource_wait = 2;
17681
17682 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17683 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17684 .expect("no io error acquiring directly")
17685 {
17686 crate::cache::AcquireOutcome::Acquired(g) => g,
17687 crate::cache::AcquireOutcome::Busy(b) => {
17688 panic!("expected the direct acquire to win the lease first: {b:?}")
17689 }
17690 };
17691
17692 let mut state = RunState::new(
17693 repo.clone(),
17694 "main".to_owned(),
17695 "deadbeef".to_owned(),
17696 "task".to_owned(),
17697 config,
17698 );
17699 state.candidates = vec![Candidate {
17700 index: 0,
17701 label: 'A',
17702 agent: "alpha".to_owned(),
17703 branch: "does-not-exist".to_owned(),
17704 worktree: repo.clone(),
17705 summary: String::new(),
17706 stat: String::new(),
17707 files: 0,
17708 commits: 0,
17709 empty: false,
17710 failed: None,
17711 verified_noop: None,
17712 duration_ms: 0,
17713 folded: false,
17714 }];
17715 state.tally = Some(Tally {
17716 first_choice: BTreeMap::from([('A', 1)]),
17717 borda: BTreeMap::new(),
17718 winner: 'A',
17719 rankings: 1,
17720 unanimous_initial: true,
17721 deliberated: false,
17722 changed_votes: 0,
17723 unanimous_final: true,
17724 tie_break: None,
17725 judges: 0,
17726 present: 0,
17727 quorum: 0,
17728 met_quorum: true,
17729 uncontested: Some("only candidate A produced a change".to_owned()),
17730 });
17731 state.reviews = vec![ReviewRound {
17732 round: 1,
17733 head: "deadbeef".to_owned(),
17734 verified_head: None,
17735 verified_at: None,
17736 reviews: Vec::new(),
17737 e2e: Vec::new(),
17738 fix: None,
17739 blocking: 0,
17740 answered: 0,
17741 expected: 0,
17742 clean: true,
17743 verify_retried: false,
17744 e2e_deferred: false,
17745 e2e_defer_reason: None,
17746 progressed: false,
17747 vote_split: false,
17748 reconsideration: Vec::new(),
17749 verdict: None,
17750 }];
17751
17752 state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
17753 state.reviews[0].e2e = vec![CommandOutcome {
17754 command: state.config.verify.e2e[0].clone(),
17755 code: Some(0),
17756 output_tail: String::new(),
17757 duration_ms: 0,
17758 resource_blocked: false,
17759 }];
17760 let mut runner = Runner {
17761 state,
17762 roles: ResolvedRoles {
17763 implementers: Vec::new(),
17764 judges: Vec::new(),
17765 reviewers: Vec::new(),
17766 fixer: None,
17767 conductor: conductor(),
17768 implementer_roster: Vec::new(),
17769 judge_roster: Vec::new(),
17770 reviewer_roster: Vec::new(),
17771 },
17772 sem: Arc::new(Semaphore::new(1)),
17773 pause: Pause::new(),
17774 interrupt: Pause::new(),
17775 };
17776
17777 let started = std::time::Instant::now();
17778 runner.gate().await.expect("gate");
17779 assert!(
17780 started.elapsed() < Duration::from_secs(1),
17781 "a gate with nothing to run must never wait on a lease it never needed"
17782 );
17783 assert!(
17784 runner.state.gate_ran,
17785 "zero commands is still a real, immediate attempt"
17786 );
17787 assert!(runner.state.gate.is_empty());
17788 assert_ne!(
17789 runner.state.status,
17790 RunStatus::Blocked,
17791 "must not read as resource-blocked on a lease it never asked for"
17792 );
17793 }
17794
17795 #[cfg(unix)]
17798 #[tokio::test]
17799 async fn verification_is_standalone_inside_a_live_worker() {
17800 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17801 if std::env::var_os(CHILD).is_none() {
17802 let home = tempfile::tempdir().unwrap();
17803 let generation = crate::worker::pin_current(home.path()).unwrap();
17804 let output = std::process::Command::new(std::env::current_exe().unwrap())
17805 .args([
17806 "--exact",
17807 "graph::tests::verification_is_standalone_inside_a_live_worker",
17808 "--nocapture",
17809 ])
17810 .env(CHILD, "1")
17811 .env("MAGI_HOME", home.path())
17812 .env("MAGI_WORKER_HOME", home.path())
17813 .env("MAGI_WORKER_ID", "abcdef")
17814 .env("MAGI_WORKER_GENERATION", &generation.id)
17815 .env("MAGI_WORKER_EXE", &generation.executable)
17816 .env("MAGI_RUN", "verification-parent")
17817 .env("MAGI_NODE", "gate")
17818 .env("MAGI_SEAT", "worker-fixture")
17819 .env("MAGI_PROCESS_OWNER", "fixture-owner")
17820 .env("MAGI_NO_AUTOUPDATE", "0")
17821 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17822 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17823 .env(
17824 "VERIFY_EXPECT_TOOLCHAIN",
17825 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17826 )
17827 .env(
17828 "VERIFY_EXPECT_CACHE",
17829 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17830 )
17831 .output()
17832 .unwrap();
17833 assert!(
17834 output.status.success()
17835 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17836 "{}\n{}",
17837 String::from_utf8_lossy(&output.stdout),
17838 String::from_utf8_lossy(&output.stderr)
17839 );
17840 return;
17841 }
17842 let home = crate::run::home();
17843 let directory = home.join("workers/jobs/abcdef");
17844 std::fs::create_dir_all(&directory).unwrap();
17845 let job = crate::worker::Job {
17846 id: "abcdef".into(),
17847 sequence: 1,
17848 kind: "task".into(),
17849 key: "task:verification".into(),
17850 payload: serde_json::json!({}),
17851 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17852 version: Some(env!("CARGO_PKG_VERSION").into()),
17853 status: crate::worker::Status::Running,
17854 result: None,
17855 pid: Some(std::process::id()),
17856 created_at: 0,
17857 finished_at: None,
17858 };
17859 std::fs::write(
17860 directory.join("job.json"),
17861 serde_json::to_vec(&job).unwrap(),
17862 )
17863 .unwrap();
17864 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17865 let assert_worker = || {
17866 assert_eq!(
17867 crate::worker::current_job(&home).unwrap().unwrap().id,
17868 job.id
17869 );
17870 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17871 assert_eq!(
17872 crate::worker::helper_executable(&home).unwrap(),
17873 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17874 );
17875 assert!(
17876 crate::updater::Checker::new(&crate::config::Update {
17877 mode: crate::config::UpdateMode::Notify,
17878 interval: None,
17879 auto_restart: false,
17880 })
17881 .is_none(),
17882 "the driver remains pinned, even while tests are standalone"
17883 );
17884 };
17885 assert_worker();
17886 let repo = home.join("repo");
17887 std::fs::create_dir_all(&repo).unwrap();
17888 init_repo(&repo);
17889 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17892 test -z "${MAGI_WORKER_GENERATION+x}" &&
17893 test -z "${MAGI_WORKER_HOME+x}" &&
17894 test -z "${MAGI_WORKER_EXE+x}" &&
17895 test "$MAGI_HOME/repo" -ef . &&
17896 test "$MAGI_RUN" = verification-parent &&
17897 test "$MAGI_NODE" = gate &&
17898 test "$MAGI_SEAT" = worker-fixture &&
17899 test -n "$MAGI_PROCESS_OWNER" &&
17900 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17901 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17902 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17903 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17904 assert!(
17906 !std::process::Command::new("sh")
17907 .args(["-c", probe])
17908 .current_dir(&repo)
17909 .status()
17910 .unwrap()
17911 .success()
17912 );
17913 let unrelated_home = tempfile::tempdir().unwrap();
17914 assert!(
17915 crate::worker::current_job(unrelated_home.path()).is_err(),
17916 "an inherited worker id cannot be resolved in a unit fixture's home"
17917 );
17918 let mut runner = runner_at(RunStatus::Gating);
17919 runner.state.repo = repo.clone();
17920 runner.state.config.disk.min_free_bytes = 0;
17921 runner.state.config.coordination = crate::config::test_coordination();
17922 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17923 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17924 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17925 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17926 let winner = Candidate {
17927 index: 0,
17928 label: 'A',
17929 agent: "fixture".into(),
17930 branch: "main".into(),
17931 worktree: repo.clone(),
17932 summary: String::new(),
17933 stat: String::new(),
17934 files: 0,
17935 commits: 0,
17936 empty: false,
17937 failed: None,
17938 verified_noop: None,
17939 duration_ms: 0,
17940 folded: false,
17941 };
17942 let (e2e, retried, pids) = run_e2e_with_retry(
17943 &mut runner.state,
17944 &["sh".into(), "-c".into()],
17945 &[format!("{probe} && printf initial")],
17946 &repo,
17947 Duration::from_secs(10),
17948 "initial",
17949 )
17950 .await;
17951 assert!(!retried && pids.is_empty());
17952 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17953 assert_eq!(e2e[0].output_tail, "initial");
17954 let gate = runner.run_gate(&winner).await.unwrap();
17955 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17956 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17959 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17960 passed: false,
17961 outcomes: Vec::new(),
17962 });
17963 runner.state.save().unwrap();
17964 let mut resumed = RunState::load(&runner.state.id).unwrap();
17965 let wait = crate::repo_lock::WaitPolicy::default();
17966 assert!(matches!(
17967 check_gate_repair_e2e(&mut resumed, &winner, &wait)
17968 .await
17969 .unwrap(),
17970 GateFix::Retry
17971 ));
17972 let saved = RunState::load(&resumed.id).unwrap();
17973 let repair = saved.gate_repair_e2e.unwrap();
17974 assert!(repair.passed);
17975 assert_eq!(repair.outcomes.len(), 1);
17976 assert!(repair.outcomes[0].ok());
17977 let landing = run_landing_gate(&mut resumed, &winner, &wait)
17978 .await
17979 .unwrap();
17980 assert_eq!(landing.len(), 1);
17981 assert!(landing[0].ok(), "{landing:?}");
17982 assert_eq!(
17983 std::fs::read_to_string(repo.join("phases")).unwrap(),
17984 "pre\ngate\ne2e\npre\ngate\n"
17985 );
17986 assert_worker();
17987 }
17988
17989 #[cfg(unix)]
17990 #[tokio::test]
17991 async fn verification_keeps_the_panic_and_the_full_output() {
17992 crate::run::pin_test_home();
17993 let tmp = tempfile::tempdir().unwrap();
17994 let mut config = Config::default();
17995 config.disk.min_free_bytes = 0;
17996 let mut state = RunState::new(
17997 tmp.path().to_owned(),
17998 "main".into(),
17999 "base".into(),
18000 "verification evidence".into(),
18001 config,
18002 );
18003 let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
18004 i=0; while [ $i -lt 20000 ]; do echo \" $i: 0xabc - <a::b>\"; \
18005 echo ' at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
18006 let mut names = Vec::new();
18007 for _ in 0..2 {
18008 let (outcomes, _) = run_commands(
18009 &mut state,
18010 "gate",
18011 "gate",
18012 0,
18013 &["sh".into(), "-c".into()],
18014 &[script.into()],
18015 tmp.path(),
18016 Duration::from_secs(60),
18017 )
18018 .await;
18019 let tail = &outcomes[0].output_tail;
18020 assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
18021 assert!(tail.contains("boom assertion"));
18022 assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
18023 let line = tail.lines().next().unwrap();
18024 names.push(line.to_owned());
18025 }
18026 assert_ne!(names[0], names[1], "the second run must not overwrite");
18027 let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
18028 assert!(full.len() > 500_000 && full.contains("boom assertion"));
18029 assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
18030
18031 std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
18033 std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
18034 let (outcomes, _) = run_commands(
18035 &mut state,
18036 "gate",
18037 "gate",
18038 0,
18039 &["sh".into(), "-c".into()],
18040 &["echo failure-output; exit 7".into()],
18041 tmp.path(),
18042 Duration::from_secs(10),
18043 )
18044 .await;
18045 assert_eq!(outcomes[0].code, Some(7));
18046 assert!(state.events.iter().any(|e| {
18047 e.node == "gate" && e.message.contains("could not save the full gate output:")
18048 }));
18049 }
18050
18051 #[cfg(unix)]
18052 #[tokio::test]
18053 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
18054 crate::run::pin_test_home();
18055 let tmp = tempfile::tempdir().unwrap();
18056 let mut config = Config::default();
18057 config.disk.min_free_bytes = 0;
18058 let mut state = RunState::new(
18059 tmp.path().to_owned(),
18060 "main".into(),
18061 "base".into(),
18062 "verification evidence".into(),
18063 config,
18064 );
18065 let (outcomes, pids) = run_commands(
18066 &mut state,
18067 "gate",
18068 "gate",
18069 1,
18070 &["sh".into(), "-c".into()],
18071 &[
18072 "printf before-timeout; exec sleep 30".into(),
18073 "kill -TERM $$".into(),
18074 "exit 7".into(),
18075 "printf success".into(),
18076 ],
18077 tmp.path(),
18078 Duration::from_secs(1),
18079 )
18080 .await;
18081 wait_for_timed_out_children_to_die(&pids).await;
18082 assert_eq!(
18083 pids.len(),
18084 1,
18085 "unleased timeout retains its death-check PID"
18086 );
18087 assert_eq!(outcomes.len(), 4);
18088 assert_eq!(outcomes[0].code, None);
18089 assert!(outcomes[0].output_tail.contains("before-timeout"));
18090 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
18091 assert_eq!(outcomes[1].code, None);
18092 assert!(outcomes[1].output_tail.contains("command terminated:"));
18093 assert!(!outcomes[1].output_tail.contains("timed out"));
18094 assert_eq!(outcomes[2].code, Some(7));
18095 assert!(!outcomes[2].output_tail.contains("timed out"));
18096 assert_eq!(outcomes[3].code, Some(0));
18097 assert_eq!(outcomes[3].output_tail, "success");
18098 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
18099 }
18100
18101 #[tokio::test]
18111 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
18112 crate::run::pin_test_home();
18113
18114 let tmp = tempfile::tempdir().expect("tempdir");
18115 let repo = tmp.path().join("repo");
18116 std::fs::create_dir_all(&repo).unwrap();
18117 init_repo(&repo);
18118
18119 let mut config = Config::default();
18120 config.verify.gate = vec![
18121 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
18122 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
18123 .to_owned(),
18124 ];
18125
18126 let mut state = RunState::new(
18127 repo.clone(),
18128 "main".to_owned(),
18129 "deadbeef".to_owned(),
18130 "task".to_owned(),
18131 config,
18132 );
18133 let run_id = state.id.clone();
18134 state.candidates = vec![Candidate {
18135 index: 0,
18136 label: 'A',
18137 agent: "alpha".to_owned(),
18138 branch: "does-not-exist".to_owned(),
18139 worktree: repo.clone(),
18140 summary: String::new(),
18141 stat: String::new(),
18142 files: 0,
18143 commits: 0,
18144 empty: false,
18145 failed: None,
18146 verified_noop: None,
18147 duration_ms: 0,
18148 folded: false,
18149 }];
18150 state.tally = Some(Tally {
18151 first_choice: BTreeMap::from([('A', 1)]),
18152 borda: BTreeMap::new(),
18153 winner: 'A',
18154 rankings: 1,
18155 unanimous_initial: true,
18156 deliberated: false,
18157 changed_votes: 0,
18158 unanimous_final: true,
18159 tie_break: None,
18160 judges: 0,
18161 present: 0,
18162 quorum: 0,
18163 met_quorum: true,
18164 uncontested: Some("only candidate A produced a change".to_owned()),
18165 });
18166 state.reviews = vec![ReviewRound {
18167 round: 1,
18168 head: "deadbeef".to_owned(),
18169 verified_head: None,
18170 verified_at: None,
18171 reviews: Vec::new(),
18172 e2e: Vec::new(),
18173 fix: None,
18174 blocking: 0,
18175 answered: 0,
18176 expected: 0,
18177 clean: true,
18178 verify_retried: false,
18179 e2e_deferred: false,
18180 e2e_defer_reason: None,
18181 progressed: false,
18182 vote_split: false,
18183 reconsideration: Vec::new(),
18184 verdict: None,
18185 }];
18186
18187 let mut runner = Runner {
18188 state,
18189 roles: ResolvedRoles {
18190 implementers: Vec::new(),
18191 judges: Vec::new(),
18192 reviewers: Vec::new(),
18193 fixer: None,
18194 conductor: conductor(),
18195 implementer_roster: Vec::new(),
18196 judge_roster: Vec::new(),
18197 reviewer_roster: Vec::new(),
18198 },
18199 sem: Arc::new(Semaphore::new(1)),
18200 pause: Pause::new(),
18201 interrupt: Pause::new(),
18202 };
18203
18204 let started_marker = repo.join("started.marker");
18205 let release_marker = repo.join("release.marker");
18206 let poller = tokio::spawn(async move {
18207 for _ in 0..100 {
18212 if started_marker.exists()
18213 && let Ok(s) = crate::run::RunState::load(&run_id)
18214 && let Some(a) = s.active.get("gate")
18215 {
18216 std::fs::write(&release_marker, b"go").expect("release marker");
18217 return Some(a.clone());
18218 }
18219 tokio::time::sleep(Duration::from_millis(50)).await;
18220 }
18221 None
18222 });
18223
18224 runner.gate().await.expect("gate");
18225 let captured = poller.await.expect("poller task");
18226 let captured = captured.expect(
18227 "the poller never saw a `gate` task entry in run.json while the command was \
18228 still blocked on its own release marker",
18229 );
18230
18231 assert_eq!(captured.task.as_deref(), Some("gate"));
18232 assert_eq!(captured.node, "gate");
18233 assert_eq!(captured.index, Some(1));
18234 assert_eq!(captured.total, Some(1));
18235 assert!(
18236 captured
18237 .command
18238 .as_deref()
18239 .is_some_and(|c| c.contains("started.marker")),
18240 "{captured:?}"
18241 );
18242
18243 assert!(
18244 runner.state.active.is_empty(),
18245 "the entry must be cleared once the command actually finished: {:?}",
18246 runner.state.active
18247 );
18248 assert!(runner.state.gate_ran);
18249 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18250 }
18251
18252 #[tokio::test]
18255 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
18256 use crate::verdict::{Finding, ReviewVote, Severity};
18257 crate::run::pin_test_home();
18258 let tmp = tempfile::tempdir().expect("tempdir");
18259 let repo = tmp.path().join("repo");
18260 std::fs::create_dir_all(&repo).unwrap();
18261 init_repo(&repo);
18262
18263 for (severity, vote, expect) in [
18264 (Severity::Major, ReviewVote::Reject, true),
18265 (Severity::Minor, ReviewVote::Reject, false),
18266 (Severity::Major, ReviewVote::Approve, false),
18267 ] {
18268 let mut round = review_round(false, 1, 1, 1, false, true);
18269 round.reviews = vec![ReviewRecord {
18270 reviewer: 1,
18271 agent: "alpha".to_owned(),
18272 summary: String::new(),
18273 findings: vec![Finding {
18274 id: "R1-1-1".to_owned(),
18275 severity,
18276 file: None,
18277 line: None,
18278 title: "t".to_owned(),
18279 detail: String::new(),
18280 }],
18281 vote: Some(vote),
18282 failed: None,
18283 duration_ms: 0,
18284 attempts: 0,
18285 }];
18286 let mut state = RunState::new(
18287 repo.clone(),
18288 "main".to_owned(),
18289 "deadbeef".to_owned(),
18290 "task".to_owned(),
18291 Config::default(),
18292 );
18293 state.reviews = vec![round];
18294 let mut runner = Runner {
18295 state,
18296 roles: ResolvedRoles {
18297 implementers: Vec::new(),
18298 judges: Vec::new(),
18299 reviewers: Vec::new(),
18300 fixer: None,
18301 conductor: conductor(),
18302 implementer_roster: Vec::new(),
18303 judge_roster: Vec::new(),
18304 reviewer_roster: Vec::new(),
18305 },
18306 sem: Arc::new(Semaphore::new(1)),
18307 pause: Pause::new(),
18308 interrupt: Pause::new(),
18309 };
18310 let shell = runner.state.config.shell();
18311 runner
18312 .stop_reviewing("round budget spent", &shell, &repo)
18313 .await
18314 .expect("stop_reviewing");
18315 assert_eq!(runner.state.status, RunStatus::Gating);
18316 assert_eq!(
18317 runner.state.contested_handoff.is_some(),
18318 expect,
18319 "{severity:?} + {vote:?}"
18320 );
18321 }
18322 }
18323
18324 #[tokio::test]
18337 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
18338 crate::run::pin_test_home();
18339 let home = crate::run::home();
18340
18341 let tmp = tempfile::tempdir().expect("tempdir");
18342 let repo = tmp.path().join("repo");
18343 std::fs::create_dir_all(&repo).unwrap();
18344 init_repo(&repo);
18345 let head = crate::git::rev_parse(&repo, "HEAD")
18346 .await
18347 .expect("rev-parse");
18348 let cache_dir = tmp.path().join("target");
18351
18352 let mut config = Config::default();
18353 config.verify.e2e = vec![format!(
18354 "CARGO_TARGET_DIR='{}' test -f README.md",
18355 cache_dir.display()
18356 )];
18357 config.graph.review_rounds = 1;
18358 config.graph.timeout_verify = Some(2);
18361 config.graph.timeout_resource_wait = 2;
18362
18363 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18364 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18365 .expect("no io error acquiring directly")
18366 {
18367 crate::cache::AcquireOutcome::Acquired(g) => g,
18368 crate::cache::AcquireOutcome::Busy(b) => {
18369 panic!("expected the direct acquire to win the lease first: {b:?}")
18370 }
18371 };
18372
18373 let mut state = RunState::new(
18374 repo.clone(),
18375 "main".to_owned(),
18376 head.clone(),
18377 "task".to_owned(),
18378 config,
18379 );
18380 state.candidates = vec![Candidate {
18381 index: 0,
18382 label: 'A',
18383 agent: "alpha".to_owned(),
18384 branch: "does-not-exist".to_owned(),
18385 worktree: repo.clone(),
18386 summary: String::new(),
18387 stat: String::new(),
18388 files: 0,
18389 commits: 0,
18390 empty: false,
18391 failed: None,
18392 verified_noop: None,
18393 duration_ms: 0,
18394 folded: false,
18395 }];
18396 state.tally = Some(Tally {
18397 first_choice: BTreeMap::from([('A', 1)]),
18398 borda: BTreeMap::new(),
18399 winner: 'A',
18400 rankings: 1,
18401 unanimous_initial: true,
18402 deliberated: false,
18403 changed_votes: 0,
18404 unanimous_final: true,
18405 tie_break: None,
18406 judges: 0,
18407 present: 0,
18408 quorum: 0,
18409 met_quorum: true,
18410 uncontested: Some("only candidate A produced a change".to_owned()),
18411 });
18412 state.reviews = vec![ReviewRound {
18416 round: 1,
18417 head: head.clone(),
18418 verified_head: None,
18419 verified_at: None,
18420 reviews: Vec::new(),
18421 e2e: Vec::new(),
18422 fix: None,
18423 blocking: 1,
18424 answered: 1,
18425 expected: 1,
18426 clean: false,
18427 verify_retried: false,
18428 e2e_deferred: true,
18429 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
18430 progressed: false,
18431 vote_split: false,
18432 reconsideration: Vec::new(),
18433 verdict: None,
18434 }];
18435
18436 let mut runner = Runner {
18437 state,
18438 roles: ResolvedRoles {
18439 implementers: Vec::new(),
18440 judges: Vec::new(),
18441 reviewers: Vec::new(),
18442 fixer: None,
18443 conductor: conductor(),
18444 implementer_roster: Vec::new(),
18445 judge_roster: Vec::new(),
18446 reviewer_roster: Vec::new(),
18447 },
18448 sem: Arc::new(Semaphore::new(1)),
18449 pause: Pause::new(),
18450 interrupt: Pause::new(),
18451 };
18452
18453 let shell = runner.state.config.shell();
18454 runner
18455 .stop_reviewing("round budget spent", &shell, &repo)
18456 .await
18457 .expect("stop_reviewing");
18458
18459 let last = runner.state.reviews.last().expect("round record");
18460 assert_eq!(
18461 last.e2e_status(),
18462 E2eStatus::ResourceBlocked,
18463 "the shared cache is still held; the attempt must read as blocked, not deferred or \
18464 failed: {last:?}"
18465 );
18466 assert_eq!(
18467 last.verified_head.as_deref(),
18468 Some(head.as_str()),
18469 "which commit this attempt targeted is known even though nothing finished checking \
18470 it"
18471 );
18472 let first_attempt_at = last
18473 .verified_at
18474 .expect("when this attempt ran is known too");
18475 assert_ne!(
18476 runner.state.status,
18477 RunStatus::Blocked,
18478 "contention is evidence about the machine, not the patch — it must not settle the \
18479 run as blocked: {:?}",
18480 runner.state.status
18481 );
18482 assert!(
18483 !runner
18484 .state
18485 .events
18486 .iter()
18487 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
18488 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
18489 runner.state.events
18490 );
18491
18492 runner
18497 .stop_reviewing("round budget spent", &shell, &repo)
18498 .await
18499 .expect("stop_reviewing retry");
18500 assert_eq!(
18501 runner.state.reviews.len(),
18502 1,
18503 "no new round was started: {:?}",
18504 runner.state.reviews
18505 );
18506 let last = runner.state.reviews.last().expect("round record");
18507 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
18508 assert!(
18509 last.verified_at.expect("still known") > first_attempt_at,
18510 "a second reentry must be a fresh attempt, not a stale copy of the first"
18511 );
18512 assert_ne!(runner.state.status, RunStatus::Blocked);
18513
18514 held.release();
18515 }
18516
18517 #[cfg(unix)]
18518 #[tokio::test]
18519 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
18520 ask_test_home();
18521 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
18522 let tmp = tempfile::tempdir().unwrap();
18523 let repo = tmp.path().join("repo");
18524 std::fs::create_dir(&repo).unwrap();
18525 init_repo(&repo);
18526 let run_git = |args: &[&str]| {
18527 let out = std::process::Command::new("git")
18528 .args(args)
18529 .current_dir(&repo)
18530 .output()
18531 .unwrap();
18532 assert!(
18533 out.status.success(),
18534 "{args:?}: {}",
18535 String::from_utf8_lossy(&out.stderr)
18536 );
18537 };
18538 let origin = tmp.path().join("origin.git");
18539 run_git(&["init", "--bare", origin.to_str().unwrap()]);
18540 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
18541 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
18542 run_git(&["add", "spec.rs"]);
18543 run_git(&["commit", "-m", "initial spec"]);
18544 run_git(&["push", "origin", "main"]);
18545 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
18546 run_git(&["checkout", "-b", "fix/review"]);
18547 std::fs::write(
18548 repo.join("check.rs"),
18549 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
18550 )
18551 .unwrap();
18552 run_git(&["add", "check.rs"]);
18553 run_git(&["commit", "-m", "use spec"]);
18554 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
18555 run_git(&["checkout", "main"]);
18556 std::fs::write(
18557 repo.join("spec.rs"),
18558 "struct AgentSpec { max_turn_secs: u64 }\n",
18559 )
18560 .unwrap();
18561 run_git(&["add", "spec.rs"]);
18562 run_git(&["commit", "-m", "extend spec"]);
18563 run_git(&["push", "origin", "main"]);
18564 run_git(&["checkout", "fix/review"]);
18565
18566 let mut runner = runner_at(RunStatus::Reviewing);
18567 runner.state.repo = repo.clone();
18568 runner.state.base_commit = base;
18569 runner.state.pr_head = Some("fix/review".into());
18570 runner.state.config.graph.review_rounds = max_rounds;
18571 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
18572 runner.state.config.graph.sessions = false;
18573 runner.state.config.disk.min_free_bytes = 0;
18574 runner.state.config.verify.e2e =
18575 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
18576 runner.state.candidates = vec![Candidate {
18577 index: 0,
18578 label: 'A',
18579 agent: "fixture".into(),
18580 branch: "fix/review".into(),
18581 worktree: repo.clone(),
18582 summary: String::new(),
18583 stat: String::new(),
18584 files: 1,
18585 commits: 1,
18586 empty: false,
18587 failed: None,
18588 verified_noop: None,
18589 duration_ms: 0,
18590 folded: false,
18591 }];
18592 runner.state.tally = Some(Tally {
18593 first_choice: BTreeMap::new(),
18594 borda: BTreeMap::new(),
18595 winner: 'A',
18596 rankings: 0,
18597 unanimous_initial: true,
18598 deliberated: false,
18599 changed_votes: 0,
18600 unanimous_final: true,
18601 tie_break: None,
18602 judges: 0,
18603 present: 0,
18604 quorum: 0,
18605 met_quorum: true,
18606 uncontested: Some("review-only".into()),
18607 });
18608 let mut last = review_round(false, 0, 1, 1, false, false);
18609 last.head = reviewed.clone();
18610 last.e2e_deferred = true;
18611 last.e2e[0].resource_blocked = true;
18612 last.e2e[0].output_tail = "waiting for the shared build cache".into();
18613 if max_rounds == 1 {
18614 last.e2e.clear();
18615 }
18616 runner.state.reviews = vec![last];
18617 let calls = tmp.path().join("fix-calls");
18618 let mut fixer = spec("repair");
18619 fixer
18620 .env
18621 .insert("FIX_CALLS".into(), calls.display().to_string());
18622 fixer.command = vec!["sh".into(), "-c".into(), r#"
18623 echo fix >> "$FIX_CALLS"
18624 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
18625 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
18626 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
18627 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
18628 "#.into()];
18629 let mut reviewer = spec("reviewer");
18630 reviewer.command = vec![
18631 "sh".into(),
18632 "-c".into(),
18633 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
18634 .into(),
18635 ];
18636 runner.roles.fixer = Some(vec![fixer]);
18637 runner.roles.reviewers = vec![reviewer];
18638
18639 runner.sync_to_base().await.unwrap();
18640 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
18641 assert_ne!(rebased, reviewed);
18642 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
18643 if reentry {
18644 let shell = runner.state.config.shell();
18647 runner
18648 .stop_unresolved_review("deferred verification", &shell, &repo)
18649 .await
18650 .unwrap();
18651 assert!(runner.catchup_repairable());
18652 assert!(!calls.exists());
18653 runner.state = RunState::load(&runner.state.id).unwrap();
18654 assert!(runner.catchup_repairable());
18655 }
18656 runner.review_loop().await.unwrap();
18657 let first = &runner.state.reviews[0];
18658 assert_eq!(first.head, reviewed);
18659 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
18660 assert!(
18661 first.verify_retried,
18662 "a reproduced build failure gets exactly one retry: {:?}",
18663 first.e2e
18664 );
18665 assert!(first.e2e[0].output_tail.contains("E0063"));
18666 assert!(!runner.catchup_repairable());
18667 if max_rounds == 2 {
18668 assert_eq!(
18669 runner.state.status,
18670 RunStatus::Gating,
18671 "{}",
18672 crate::report::run(&runner.state)
18673 );
18674 assert_eq!(runner.state.reviews.len(), 2);
18675 assert!(first.fix.as_ref().unwrap().committed);
18676 assert!(runner.state.reviews[1].clean);
18677 assert_eq!(
18678 runner
18679 .state
18680 .reviews
18681 .iter()
18682 .filter(|r| r.fix.is_some())
18683 .count(),
18684 1
18685 );
18686 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
18687 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
18688 } else {
18689 assert_eq!(runner.state.status, RunStatus::Blocked);
18690 assert!(first.fix.is_none());
18691 assert!(!calls.exists());
18692 let recovery = crate::recovery::assess(
18693 "prior",
18694 Some(&runner.state),
18695 &runner.landing_base(),
18696 crate::recovery::Relation::Diverged,
18697 );
18698 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
18699 let reasons = recovery.reasons.join("\n");
18700 assert!(
18701 reasons.contains(&reviewed) && reasons.contains(&rebased),
18702 "{reasons}"
18703 );
18704 assert!(
18705 reasons.contains("base sync to")
18706 && reasons.contains("additional review is required"),
18707 "{reasons}"
18708 );
18709 }
18710 }
18711 }
18712
18713 #[tokio::test]
18725 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
18726 crate::run::pin_test_home();
18727 let home = crate::run::home();
18728
18729 let tmp = tempfile::tempdir().expect("tempdir");
18730 let repo = tmp.path().join("repo");
18731 std::fs::create_dir_all(&repo).unwrap();
18732 init_repo(&repo);
18733 let head = crate::git::rev_parse(&repo, "HEAD")
18734 .await
18735 .expect("rev-parse");
18736 let cache_dir = tmp.path().join("target");
18737
18738 let mut config = Config::default();
18739 config.verify.e2e = vec![format!(
18740 "CARGO_TARGET_DIR='{}' test -f README.md",
18741 cache_dir.display()
18742 )];
18743 config.graph.review_rounds = 1;
18744 config.graph.timeout_verify = Some(2);
18745 config.graph.timeout_resource_wait = 2;
18746
18747 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18748 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18749 .expect("no io error acquiring directly")
18750 {
18751 crate::cache::AcquireOutcome::Acquired(g) => g,
18752 crate::cache::AcquireOutcome::Busy(b) => {
18753 panic!("expected the direct acquire to win the lease first: {b:?}")
18754 }
18755 };
18756
18757 let mut state = RunState::new(
18758 repo.clone(),
18759 "main".to_owned(),
18760 head.clone(),
18761 "task".to_owned(),
18762 config,
18763 );
18764 state.candidates = vec![Candidate {
18765 index: 0,
18766 label: 'A',
18767 agent: "alpha".to_owned(),
18768 branch: "does-not-exist".to_owned(),
18769 worktree: repo.clone(),
18770 summary: String::new(),
18771 stat: String::new(),
18772 files: 0,
18773 commits: 0,
18774 empty: false,
18775 failed: None,
18776 verified_noop: None,
18777 duration_ms: 0,
18778 folded: false,
18779 }];
18780 state.tally = Some(Tally {
18781 first_choice: BTreeMap::from([('A', 1)]),
18782 borda: BTreeMap::new(),
18783 winner: 'A',
18784 rankings: 1,
18785 unanimous_initial: true,
18786 deliberated: false,
18787 changed_votes: 0,
18788 unanimous_final: true,
18789 tie_break: None,
18790 judges: 0,
18791 present: 0,
18792 quorum: 0,
18793 met_quorum: true,
18794 uncontested: Some("only candidate A produced a change".to_owned()),
18795 });
18796 state.reviews = vec![ReviewRound {
18800 round: 1,
18801 head: head.clone(),
18802 verified_head: Some(head.clone()),
18803 verified_at: Some(jiff::Timestamp::now()),
18804 reviews: Vec::new(),
18805 e2e: vec![CommandOutcome {
18806 command: format!(
18807 "CARGO_TARGET_DIR='{}' test -f README.md",
18808 cache_dir.display()
18809 ),
18810 code: None,
18811 output_tail: "waiting for the shared build cache".to_owned(),
18812 duration_ms: 0,
18813 resource_blocked: true,
18814 }],
18815 fix: None,
18816 blocking: 1,
18817 answered: 1,
18818 expected: 1,
18819 clean: false,
18820 verify_retried: false,
18821 e2e_deferred: false,
18822 e2e_defer_reason: None,
18823 progressed: false,
18824 vote_split: false,
18825 reconsideration: Vec::new(),
18826 verdict: None,
18827 }];
18828
18829 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18830 let mut runner = Runner {
18831 state,
18832 roles: ResolvedRoles {
18833 implementers: Vec::new(),
18834 judges: Vec::new(),
18835 reviewers: Vec::new(),
18836 fixer: None,
18837 conductor: conductor(),
18838 implementer_roster: Vec::new(),
18839 judge_roster: Vec::new(),
18840 reviewer_roster: Vec::new(),
18841 },
18842 sem: Arc::new(Semaphore::new(1)),
18843 pause: Pause::new(),
18844 interrupt: Pause::new(),
18845 };
18846
18847 runner.review_loop().await.expect("review_loop");
18852
18853 assert_eq!(
18854 runner.state.reviews.len(),
18855 1,
18856 "no new round was started on top of the unresolved one: {:?}",
18857 runner.state.reviews
18858 );
18859 let last = &runner.state.reviews[0];
18860 assert_eq!(
18861 last.e2e_status(),
18862 E2eStatus::ResourceBlocked,
18863 "still contended: {last:?}"
18864 );
18865 assert!(
18866 last.verified_at.expect("still known") > first_attempt_at,
18867 "review_loop must have actually retried the check, not left it exactly as found"
18868 );
18869 assert_ne!(
18870 runner.state.status,
18871 RunStatus::Blocked,
18872 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18873 runner.state.status
18874 );
18875
18876 held.release();
18877 }
18878
18879 #[tokio::test]
18880 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18881 crate::run::pin_test_home();
18882 for status in [RunStatus::Blocked, RunStatus::Gating] {
18883 let mut runner = runner_at(status);
18884 runner.state.config.graph.land = true;
18885 runner.state.gate_ran = true;
18886 runner.state.merge = Some(MergeOutcome {
18887 mode: MergeMode::Pr,
18888 ok: false,
18889 detail: "the fixer produced no commit".into(),
18890 empty: false,
18891 });
18892 let url = "https://example.invalid/o/r/pull/1";
18893 runner.state.pr = Some(crate::run::PrRecord {
18894 url: url.into(),
18895 number: 1,
18896 state: "open".into(),
18897 checks: "red".into(),
18898 round: 1,
18899 rounds: 1,
18900 red_at_merge: Vec::new(),
18901 });
18902 runner.state.save().unwrap();
18903 runner.execute_graph().await.unwrap();
18907 assert_eq!(runner.state.status, RunStatus::Blocked);
18908 assert!(runner.state.land_sync_pending);
18909 assert!(runner.state.events.iter().any(|event|
18910 event.node == "land" && event.message == format!("watching {url}")));
18911 assert_eq!(
18912 runner.state.merge.as_ref().unwrap().detail,
18913 "the fixer produced no commit"
18914 );
18915 }
18916 }
18917
18918 #[tokio::test]
18919 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18920 crate::run::pin_test_home();
18921 let tmp = tempfile::tempdir().expect("tempdir");
18922 let repo = tmp.path().join("repo");
18923 std::fs::create_dir_all(&repo).unwrap();
18924 init_repo(&repo);
18925
18926 let mut config = Config::default();
18927 config.merge.mode = MergeMode::Pr;
18928 config.graph.land = true;
18929 config.graph.land_approval = false;
18930
18931 let mut state = RunState::new(
18932 repo.clone(),
18933 "main".to_owned(),
18934 "deadbeef".to_owned(),
18935 "task".to_owned(),
18936 config,
18937 );
18938 state.candidates = vec![Candidate {
18939 index: 0,
18940 label: 'A',
18941 agent: "alpha".to_owned(),
18942 branch: "does-not-exist".to_owned(),
18943 worktree: repo.clone(),
18944 summary: String::new(),
18945 stat: String::new(),
18946 files: 0,
18947 commits: 0,
18948 empty: false,
18949 failed: None,
18950 verified_noop: None,
18951 duration_ms: 0,
18952 folded: false,
18953 }];
18954 state.tally = Some(Tally {
18955 first_choice: BTreeMap::from([('A', 1)]),
18956 borda: BTreeMap::new(),
18957 winner: 'A',
18958 rankings: 1,
18959 unanimous_initial: true,
18960 deliberated: false,
18961 changed_votes: 0,
18962 unanimous_final: true,
18963 tie_break: None,
18964 judges: 0,
18965 present: 0,
18966 quorum: 0,
18967 met_quorum: true,
18968 uncontested: Some("only candidate A produced a change".to_owned()),
18969 });
18970 state.reviews = vec![ReviewRound {
18971 round: 1,
18972 head: "deadbeef".to_owned(),
18973 verified_head: None,
18974 verified_at: None,
18975 reviews: Vec::new(),
18976 e2e: Vec::new(),
18977 fix: None,
18978 blocking: 0,
18979 answered: 0,
18980 expected: 0,
18981 clean: true,
18982 verify_retried: false,
18983 e2e_deferred: false,
18984 e2e_defer_reason: None,
18985 progressed: false,
18986 vote_split: false,
18987 reconsideration: Vec::new(),
18988 verdict: None,
18989 }];
18990 state.gate = vec![CommandOutcome {
18991 command: "test".to_owned(),
18992 code: Some(0),
18993 output_tail: String::new(),
18994 duration_ms: 0,
18995 resource_blocked: false,
18996 }];
18997 state.gate_ran = true;
18998 state.status = RunStatus::Landing;
19002 state.merge = Some(MergeOutcome {
19003 mode: MergeMode::Pr,
19004 ok: true,
19005 detail: "https://example.invalid/x/y/pull/1".to_owned(),
19006 empty: false,
19007 });
19008
19009 ask_test_home();
19013 state.save().expect("save landing run");
19016 let store = ask::Questions::open();
19017 let q = ask_open_question(&store, &state.id);
19018
19019 let mut runner = Runner {
19020 state,
19021 roles: ResolvedRoles {
19022 implementers: Vec::new(),
19023 judges: Vec::new(),
19024 reviewers: Vec::new(),
19025 fixer: None,
19026 conductor: conductor(),
19027 implementer_roster: Vec::new(),
19028 judge_roster: Vec::new(),
19029 reviewer_roster: Vec::new(),
19030 },
19031 sem: Arc::new(Semaphore::new(1)),
19032 pause: Pause::new(),
19033 interrupt: Pause::new(),
19034 };
19035
19036 runner.execute().await.expect("execute");
19041
19042 assert_eq!(
19043 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19044 Some("https://example.invalid/x/y/pull/1"),
19045 "reentry must not push again or open a second pull request over the \
19046 one `land` is already watching"
19047 );
19048 assert_ne!(
19049 runner.state.status,
19050 RunStatus::Landing,
19051 "land could not actually reach the fake pull request, so it must \
19052 have given up rather than left the run silently parked forever"
19053 );
19054 assert_eq!(runner.state.status, RunStatus::Blocked);
19058 assert!(
19059 store.get(&q.id).unwrap().status.open(),
19060 "Blocked is still alive; settle_questions must have been a no-op here"
19061 );
19062 }
19063
19064 fn state_with_round(round: ReviewRound) -> RunState {
19065 let mut s = RunState::new(
19066 PathBuf::from("/repo"),
19067 "main".to_owned(),
19068 "abc1234".to_owned(),
19069 "add retries".to_owned(),
19070 Config::default(),
19071 );
19072 s.reviews = vec![round];
19073 s
19074 }
19075
19076 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
19077 crate::verdict::Finding {
19078 id: id.to_owned(),
19079 severity,
19080 file: None,
19081 line: None,
19082 title: title.to_owned(),
19083 detail: String::new(),
19084 }
19085 }
19086
19087 #[test]
19088 fn pr_body_names_open_findings_and_declined_ones() {
19089 let round = ReviewRound {
19090 round: 2,
19091 head: "deadbee".to_owned(),
19092 verified_head: None,
19093 verified_at: None,
19094 reviews: vec![ReviewRecord {
19095 attempts: 0,
19096 reviewer: 1,
19097 agent: "alpha".to_owned(),
19098 summary: String::new(),
19099 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
19100 vote: None,
19101 failed: None,
19102 duration_ms: 0,
19103 }],
19104 e2e: vec![CommandOutcome {
19105 command: "cargo test".to_owned(),
19106 code: Some(0),
19107 output_tail: String::new(),
19108 duration_ms: 0,
19109 resource_blocked: false,
19110 }],
19111 verify_retried: false,
19112 e2e_deferred: false,
19113 e2e_defer_reason: None,
19114 fix: Some(FixRecord {
19115 agent: "alpha".to_owned(),
19116 addressed: Vec::new(),
19117 rejected: vec![crate::verdict::Rejection {
19118 id: "R1-1-1".to_owned(),
19119 why: "not reachable from any caller".to_owned(),
19120 }],
19121 notes: String::new(),
19122 committed: true,
19123 failed: None,
19124 duration_ms: 0,
19125 continuation: None,
19126 }),
19127 blocking: 0,
19128 answered: 1,
19129 expected: 1,
19130 clean: false,
19131 progressed: true,
19132 vote_split: false,
19133 reconsideration: Vec::new(),
19134 verdict: None,
19135 };
19136 let state = state_with_round(round);
19137 let body = pr_message(&state, 'A').body;
19138
19139 assert!(body.contains("add retries"), "the task must still be there");
19140 assert!(body.contains("R2-1-1"), "{body}");
19141 assert!(!body.contains("unused import"), "{body}");
19142 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
19143 assert!(
19144 !body.contains("not reachable from any caller"),
19145 "the reason it was declined: {body}"
19146 );
19147 }
19148
19149 #[test]
19150 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
19151 let round = ReviewRound {
19152 round: 1,
19153 head: "deadbee".to_owned(),
19154 verified_head: None,
19155 verified_at: None,
19156 reviews: vec![ReviewRecord {
19157 attempts: 0,
19158 reviewer: 1,
19159 agent: "alpha".to_owned(),
19160 summary: String::new(),
19161 findings: Vec::new(),
19162 vote: None,
19163 failed: None,
19164 duration_ms: 0,
19165 }],
19166 e2e: Vec::new(),
19167 verify_retried: false,
19168 e2e_deferred: false,
19169 e2e_defer_reason: None,
19170 fix: None,
19171 blocking: 0,
19172 answered: 1,
19173 expected: 1,
19174 clean: true,
19175 progressed: false,
19176 vote_split: false,
19177 reconsideration: Vec::new(),
19178 verdict: None,
19179 };
19180 let state = state_with_round(round);
19181 let body = pr_message(&state, 'A').body;
19182 assert!(!body.contains("Open review findings"), "{body}");
19183 assert!(!body.contains("Declined"), "{body}");
19184 }
19185
19186 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
19187 let mut state = RunState::new(
19188 PathBuf::from("/repo"),
19189 "main".to_owned(),
19190 "abc1234".to_owned(),
19191 instruction.to_owned(),
19192 Config::default(),
19193 );
19194 state.candidates.push(Candidate {
19195 index: 0,
19196 label: 'A',
19197 agent: "alpha".to_owned(),
19198 branch: "magi/x/A".to_owned(),
19199 worktree: PathBuf::from("/wt"),
19200 summary: summary.to_owned(),
19201 stat: String::new(),
19202 files: 1,
19203 commits: 1,
19204 empty: false,
19205 failed: None,
19206 verified_noop: None,
19207 folded: false,
19208 duration_ms: 0,
19209 });
19210 state
19211 }
19212
19213 fn review_state(subjects: &[&str]) -> RunState {
19214 let mut state = state_with_summary(
19215 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
19216 "",
19217 );
19218 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19219 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
19220 state
19221 }
19222
19223 #[tokio::test]
19226 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
19227 ask_test_home();
19228 let tmp = tempfile::tempdir().unwrap();
19229 let repo = tmp.path().join("repo");
19230 std::fs::create_dir_all(&repo).unwrap();
19231 init_repo(&repo);
19232 let origin = tmp.path().join("origin.git");
19233 let g = |dir: &Path, args: &[&str]| {
19234 let out = std::process::Command::new("git")
19235 .args(args)
19236 .current_dir(dir)
19237 .quiet()
19238 .output()
19239 .expect("spawn git");
19240 assert!(
19241 out.status.success(),
19242 "git {args:?}: {}",
19243 String::from_utf8_lossy(&out.stderr)
19244 );
19245 };
19246 g(
19247 tmp.path(),
19248 &[
19249 "clone",
19250 "--bare",
19251 "-q",
19252 repo.to_str().unwrap(),
19253 origin.to_str().unwrap(),
19254 ],
19255 );
19256 g(
19257 &repo,
19258 &["remote", "add", "origin", origin.to_str().unwrap()],
19259 );
19260 let c1 = git::rev_parse(&repo, "main").await.unwrap();
19261
19262 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
19264 g(&repo, &["add", "-A"]);
19265 g(
19266 &repo,
19267 &["commit", "-q", "-m", "chore(deps): update a crate"],
19268 );
19269 g(&repo, &["push", "-q", "origin", "main"]);
19270 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
19271 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19272 g(&repo, &["add", "-A"]);
19273 g(
19274 &repo,
19275 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
19276 );
19277 g(&repo, &["checkout", "-q", "main"]);
19278
19279 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
19280 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
19281 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
19283 assert_eq!(fallback, c1);
19284
19285 let mut state = review_state(&[
19286 "chore(deps): update a crate",
19287 "fix(daemon): apply a chosen action",
19288 ]);
19289 state.repo = repo.clone();
19290 state.base_branch = "main".to_owned();
19291 state.base_commit = c1;
19292 refresh_reviewed_commits(&mut state, "feat/own").await;
19293 assert_eq!(
19294 state.reviewed_commits,
19295 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
19296 );
19297 assert_eq!(
19298 review_title(&state).as_deref(),
19299 Some("fix(daemon): apply a chosen action")
19300 );
19301 assert_eq!(
19302 leaked_subjects(&state, "feat/own").await,
19303 Some(vec!["chore(deps): update a crate".to_owned()])
19304 );
19305 state.config.merge.remote = "nowhere".to_owned();
19308 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
19309 }
19310
19311 struct SummaryFixture {
19315 _tmp: tempfile::TempDir,
19316 repo: PathBuf,
19317 reply: PathBuf,
19318 runner: Runner,
19319 }
19320
19321 fn summary_fixture() -> SummaryFixture {
19322 ask_test_home();
19323 let tmp = tempfile::tempdir().unwrap();
19324 let repo = tmp.path().join("repo");
19325 std::fs::create_dir_all(&repo).unwrap();
19326 init_repo(&repo);
19327 let origin = tmp.path().join("origin.git");
19328 let g = |args: &[&str]| {
19329 let out = std::process::Command::new("git")
19330 .args(args)
19331 .current_dir(&repo)
19332 .quiet()
19333 .output()
19334 .expect("spawn git");
19335 assert!(
19336 out.status.success(),
19337 "git {args:?}: {}",
19338 String::from_utf8_lossy(&out.stderr)
19339 );
19340 };
19341 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
19342 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
19343 g(&["push", "-q", "origin", "main"]);
19344 g(&["checkout", "-q", "-b", "feat/own"]);
19345 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
19346 g(&["add", "-A"]);
19347 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
19348 let reply = tmp.path().join("reply.txt");
19349 std::fs::write(
19350 &reply,
19351 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
19352 )
19353 .unwrap();
19354 let mut runner = runner_at(RunStatus::Gating);
19355 let mut state = review_state(&["feat(own): add the own file"]);
19356 state.repo = repo.clone();
19357 state.base_branch = "main".to_owned();
19358 state.candidates[0].branch = "feat/own".to_owned();
19359 state.candidates[0].worktree = repo.clone();
19360 let mut author = spec("alpha");
19361 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
19362 author
19363 .env
19364 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
19365 state.config.agents = vec![author];
19366 runner.state = state;
19367 SummaryFixture {
19368 _tmp: tmp,
19369 repo,
19370 reply,
19371 runner,
19372 }
19373 }
19374
19375 async fn posted(f: &mut SummaryFixture) -> PrMessage {
19376 let winner = f.runner.state.candidates[0].clone();
19377 f.runner
19378 .guarded_pr_message(&winner, None, true)
19379 .await
19380 .unwrap()
19381 .expect("an English description never parks")
19382 }
19383
19384 fn clean_round_with_minor_findings() -> ReviewRound {
19385 let mut round = passing_round(2, None);
19386 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
19387 round.reviews = vec![ReviewRecord {
19388 attempts: 0,
19389 reviewer: 1,
19390 agent: "alpha".into(),
19391 summary: "内部レビュー".into(),
19392 findings: vec![
19393 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
19394 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
19395 ],
19396 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
19397 failed: None,
19398 duration_ms: 0,
19399 }];
19400 round
19401 }
19402
19403 #[tokio::test]
19404 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
19405 for review_only in [false, true] {
19406 let mut f = summary_fixture();
19407 if !review_only {
19408 f.runner.state.candidates[0].agent = "alpha".into();
19409 }
19410 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
19411 assert!(f.runner.state.reviews[0].clean);
19412 assert!(f.runner.state.reviews[0].fix.is_none());
19413 assert!(f.runner.state.open_findings().is_empty());
19414 assert_eq!(
19415 crate::pr_outcome::required_finding_ids(&f.runner.state),
19416 ["R2-1-1", "R2-1-2"]
19417 );
19418 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19419 let evidence = crate::pr_outcome::summary_evidence(
19420 &f.runner.state,
19421 crate::pr_outcome::FinalHead::At(&head),
19422 );
19423 assert!(
19424 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
19425 );
19426 let omitted = posted(&mut f).await;
19428 assert!(omitted.body.contains("No description"));
19429 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
19430 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
19431 assert!(f.runner.state.change_summary.is_none());
19432 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
19433 let described = posted(&mut f).await;
19434 assert!(described.body.contains("improve documentation for callers"));
19435 assert!(described.body.contains("remaining edge case"));
19436 assert!(described.body.contains("## Open review findings"));
19437 assert!(!described.body.contains("未修正"));
19438 }
19439 }
19440
19441 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
19442 let mut f = summary_fixture();
19443 if !review_only {
19444 f.runner.state.candidates[0].agent = "alpha".into();
19445 }
19446 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19447 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
19448 fixed
19449 .fix
19450 .as_mut()
19451 .unwrap()
19452 .rejected
19453 .push(crate::verdict::Rejection {
19454 id: "R1-1-1".into(),
19455 why: "呼び出されないコードのため辞退".into(),
19456 });
19457 let mut clean = clean_round_with_minor_findings();
19458 clean.verified_head = Some(head.clone());
19459 f.runner.state.reviews = vec![fixed, clean];
19460 ok_gate(&mut f.runner.state, Some(&head));
19461 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
19462 f.runner.state.config.agents.push(scripted("beta", reply));
19463 f.runner.state.config.roles.language_judge =
19464 Some(crate::config::AgentChoice::One("beta".into()));
19465 f
19466 }
19467
19468 #[tokio::test]
19469 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
19470 for review_only in [false, true] {
19471 let mut f = rewrite_fixture_with_findings(review_only,
19472 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
19473 let m = posted(&mut f).await;
19474 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19475 let expected =
19476 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
19477 assert!(m.body.contains(&expected));
19478 assert!(m.body.contains("## Review rounds"));
19479 assert!(
19480 m.body
19481 .contains("Gate (`verify.gate`): passed on the final commit")
19482 );
19483 assert!(
19484 m.body
19485 .contains("Tests (`verify.e2e`): passed on the final commit")
19486 );
19487 let c = f.runner.state.change_summary.as_ref().unwrap();
19488 assert!(
19489 m.body
19490 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
19491 );
19492 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19493 assert!(crate::pr_outcome::covers_required_findings(
19494 &f.runner.state,
19495 &m.body
19496 ));
19497 assert!(m.body.contains("documentation remains incomplete"));
19498 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
19499 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
19500 assert!(f.runner.state.github_text.is_none());
19501 }
19502 }
19503
19504 #[tokio::test]
19505 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
19506 for review_only in [false, true] {
19507 for reply in [
19508 "TITLE: fix: public description\nA short English description.",
19509 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
19510 ] {
19511 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
19512 let winner = f.runner.state.candidates[0].clone();
19513 let posted = f
19514 .runner
19515 .guarded_pr_message(&winner, None, true)
19516 .await
19517 .unwrap();
19518 assert!(posted.is_none());
19519 assert!(f.runner.state.parked);
19520 let withheld = f.runner.state.github_text.as_ref().unwrap();
19521 assert!(
19522 withheld.rewrite.is_none(),
19523 "an incomplete rewrite must not be selectable"
19524 );
19525 let q = ask::Questions::open()
19526 .get(withheld.question.as_ref().unwrap())
19527 .unwrap();
19528 assert!(
19529 !q.choices
19530 .contains(&crate::github_text::USE_REWRITE.to_owned())
19531 );
19532 assert!(
19533 q.detail.contains("R1-1-1")
19534 && q.detail.contains("R2-1-1")
19535 && q.detail.contains("R2-1-2")
19536 );
19537 }
19538 }
19539 }
19540
19541 #[tokio::test]
19542 async fn japanese_reports_stay_local_in_both_public_run_modes() {
19543 for review_only in [false, true] {
19544 let mut f = summary_fixture();
19545 if !review_only {
19546 f.runner.state.candidates[0].agent = "alpha".into();
19547 }
19548 f.runner.state.instruction = "内部タスクの説明".into();
19549 f.runner.state.candidates[0].summary = "内部の候補報告".into();
19550 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19551 f.runner.state.reviews = vec![
19552 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
19553 round_with_notes(2, Some("二回目の内部修正報告です。")),
19554 passing_round(3, Some(&head)),
19555 ];
19556 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
19557 ok_gate(&mut f.runner.state, Some(&head));
19558 let captured = f.repo.join("public-prompt.txt");
19559 let author = &mut f.runner.state.config.agents[0];
19560 author.command = vec![
19561 "sh".into(),
19562 "-c".into(),
19563 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19564 ];
19565 author.env.insert(
19566 "PROMPT_FILE".into(),
19567 captured.to_string_lossy().into_owned(),
19568 );
19569 let m = posted(&mut f).await;
19570 assert!(m.body.contains("Adds `own.txt`"));
19571 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
19572 assert!(m.body.contains("R1-1-1"));
19573 assert!(
19574 m.body
19575 .contains("Gate (`verify.gate`): passed on the final commit")
19576 );
19577 assert!(
19578 m.body
19579 .contains("Tests (`verify.e2e`): passed on the final commit")
19580 );
19581 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19582 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
19583 let prompt = std::fs::read_to_string(captured).unwrap();
19584 assert!(prompt.contains(&format!(
19585 "git diff {} {head}",
19586 f.runner.state.change_summary.as_ref().unwrap().base
19587 )));
19588 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
19589 assert_eq!(
19590 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
19591 "二回目の内部修正報告です。"
19592 );
19593 }
19594 }
19595
19596 #[tokio::test]
19597 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
19598 for review_only in [false, true] {
19599 let mut f = summary_fixture();
19600 if !review_only {
19601 f.runner.state.candidates[0].agent = "alpha".into();
19602 }
19603 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
19604 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
19605 std::fs::write(&f.reply, "no usable description").unwrap();
19606 let failed = posted(&mut f).await;
19607 assert!(
19608 failed
19609 .body
19610 .contains("No description of what this change does could be generated")
19611 );
19612 assert!(failed.body.is_ascii());
19613 assert!(f.runner.state.github_text.is_none());
19614 assert!(f.runner.state.change_summary.is_none());
19615 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19616 let evidence = crate::pr_outcome::summary_evidence(
19617 &f.runner.state,
19618 crate::pr_outcome::FinalHead::At(&head),
19619 );
19620 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
19621 assert!(
19622 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
19623 .exists()
19624 );
19625 f.runner.state =
19626 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
19627 std::fs::write(
19628 &f.reply,
19629 "TITLE: feat(own): add the own file\nAdds the committed own file.",
19630 )
19631 .unwrap();
19632 let retried = posted(&mut f).await;
19633 assert!(retried.body.contains("Adds the committed own file."));
19634 assert_eq!(
19635 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
19636 "日本語の修正報告"
19637 );
19638 }
19639 }
19640
19641 #[tokio::test]
19642 async fn same_head_verification_and_findings_invalidate_public_prose() {
19643 let mut f = summary_fixture();
19644 posted(&mut f).await;
19645 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19646 let original = f
19647 .runner
19648 .state
19649 .change_summary
19650 .as_ref()
19651 .unwrap()
19652 .evidence
19653 .clone();
19654 ok_gate(&mut f.runner.state, Some(&head));
19655 std::fs::write(
19656 &f.reply,
19657 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
19658 )
19659 .unwrap();
19660 let verified = posted(&mut f).await;
19661 assert!(verified.body.contains("now verified"));
19662 assert_ne!(
19663 f.runner.state.change_summary.as_ref().unwrap().evidence,
19664 original
19665 );
19666 let mut round = round_with_notes(1, Some("内部報告"));
19667 round.reviews.push(ReviewRecord {
19668 attempts: 0,
19669 reviewer: 1,
19670 agent: "alpha".into(),
19671 summary: "内部レビュー".into(),
19672 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
19673 vote: None,
19674 failed: None,
19675 duration_ms: 0,
19676 });
19677 round
19678 .fix
19679 .as_mut()
19680 .unwrap()
19681 .rejected
19682 .push(crate::verdict::Rejection {
19683 id: "R1-1-1".into(),
19684 why: "呼び出し元から到達できない".into(),
19685 });
19686 f.runner.state.reviews = vec![round];
19687 let omitted = posted(&mut f).await;
19689 assert!(omitted.body.contains("No description"));
19690 assert!(omitted.body.contains("R1-1-1"));
19691 assert!(!omitted.body.contains("到達"));
19692 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
19693 let described = posted(&mut f).await;
19694 assert!(described.body.contains("unreachable from callers"));
19695 assert!(described.body.contains("R1-1-1"));
19696 assert!(!described.body.contains("到達"));
19697 let evidence = f
19698 .runner
19699 .state
19700 .change_summary
19701 .as_ref()
19702 .unwrap()
19703 .evidence
19704 .clone();
19705 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
19706 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
19707 assert!(posted(&mut f).await.body.contains("rationale updated"));
19708 assert_ne!(
19709 f.runner.state.change_summary.as_ref().unwrap().evidence,
19710 evidence
19711 );
19712 f.runner
19714 .state
19715 .change_summary
19716 .as_mut()
19717 .unwrap()
19718 .evidence
19719 .clear();
19720 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
19721 assert!(posted(&mut f).await.body.contains("current specification"));
19722 }
19723
19724 #[tokio::test]
19725 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
19726 let mut f = summary_fixture();
19727 let m = posted(&mut f).await;
19728 assert_eq!(m.title, "feat(own): add the own file");
19729 assert!(
19730 m.body
19731 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19732 "{}",
19733 m.body
19734 );
19735 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19736 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19737 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19738 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19739 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19740 let kept = f.runner.state.change_summary.as_ref().unwrap();
19741 assert_eq!(
19742 (kept.head.as_str(), kept.base.as_str()),
19743 (head.as_str(), base.as_str())
19744 );
19745 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19746 }
19747
19748 #[tokio::test]
19749 async fn review_only_summary_receives_the_configured_github_language() {
19750 let mut f = summary_fixture();
19751 f.runner.state.config.graph.github_language = "ja".into();
19752 let captured = f.repo.join("summary-prompt.txt");
19753 let author = &mut f.runner.state.config.agents[0];
19754 author.command = vec![
19755 "sh".into(),
19756 "-c".into(),
19757 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19758 ];
19759 author.env.insert(
19760 "PROMPT_FILE".into(),
19761 captured.to_string_lossy().into_owned(),
19762 );
19763 std::fs::write(
19764 &f.reply,
19765 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19766 )
19767 .unwrap();
19768 let m = posted(&mut f).await;
19769 let prompt = std::fs::read_to_string(captured).unwrap();
19770 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19771 assert!(!prompt.contains("in English"));
19772 assert_eq!(m.title, "feat(own): ファイルを追加");
19773 assert!(m.body.contains("own.txt を追加します。"));
19774 }
19775
19776 #[tokio::test]
19777 async fn review_fixes_come_after_the_whole_change_summary() {
19778 let mut f = summary_fixture();
19779 f.runner.state.reviews = vec![
19780 round_with_notes(1, Some("handled the PATH case")),
19781 round_with_notes(2, None),
19782 ];
19783 let m = posted(&mut f).await;
19784 let summary = m.body.find("## Summary").expect("summary");
19785 let fixes = m.body.find("## Review rounds").expect("review rounds");
19786 assert!(summary < fixes, "{}", m.body);
19787 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19788 assert!(m.body.contains("## Final verification"), "{}", m.body);
19789 }
19790
19791 #[tokio::test]
19792 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19793 let mut f = summary_fixture();
19794 let first = posted(&mut f).await;
19795 let first_head = f.runner.state.change_summary.clone().unwrap().head;
19796 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19798 let again = posted(&mut f).await;
19799 assert_eq!((again.title, again.body), (first.title, first.body));
19800
19801 let g = |args: &[&str]| {
19802 let out = std::process::Command::new("git")
19803 .args(args)
19804 .current_dir(&f.repo)
19805 .quiet()
19806 .output()
19807 .expect("spawn git");
19808 assert!(out.status.success(), "git {args:?}");
19809 };
19810 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19811 g(&["add", "-A"]);
19812 g(&["commit", "-q", "-m", "fix(own): handle more"]);
19813 let second = posted(&mut f).await;
19814 let kept = f.runner.state.change_summary.clone().unwrap();
19815 assert_ne!(kept.head, first_head);
19816 assert!(second.body.contains("Not read yet."), "{}", second.body);
19817 assert!(!second.body.contains("so the first summary is read"));
19818 assert_eq!(second.title, "feat(own): other");
19819 }
19820
19821 #[tokio::test]
19822 async fn an_unavailable_summary_is_stated_and_never_invented() {
19823 let mut f = summary_fixture();
19824 std::fs::write(&f.reply, "no title line here").unwrap();
19825 let m = posted(&mut f).await;
19826 assert!(
19827 m.body
19828 .contains("No description of what this change does could be generated"),
19829 "{}",
19830 m.body
19831 );
19832 assert!(m.body.contains("internal reports remain in local records"));
19833 assert!(f.runner.state.change_summary.is_none());
19834 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19835 let winner = f.runner.state.candidates[0].clone();
19837 let preview = f
19838 .runner
19839 .guarded_pr_message(&winner, None, false)
19840 .await
19841 .unwrap()
19842 .unwrap();
19843 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19844 }
19845
19846 #[test]
19847 fn pr_message_review_single_commit_uses_its_subject() {
19848 let state = review_state(&["feat(nats): per-role user"]);
19849 let m = pr_message(&state, 'A');
19850 assert_eq!(m.title, "feat(nats): per-role user");
19851 assert!(!m.body.contains("Review the work already"), "{}", m.body);
19852 assert!(m.body.contains("## Commits under review"), "{}", m.body);
19853 }
19854
19855 #[test]
19856 fn pr_message_review_multi_commit_takes_the_oldest() {
19857 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19858 let m = pr_message(&state, 'A');
19859 assert_eq!(m.title, "feat: the change");
19860 for s in ["feat: the change", "fix: typo", "fix: again"] {
19861 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19862 }
19863 }
19864
19865 #[test]
19866 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19867 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19868 let mut state = review_state(&[first, "fix: later fixup"]);
19869 state.candidates[0].branch = "機能/ログイン".to_owned();
19870 let m = pr_message(&state, 'A');
19871 assert!(
19872 m.title.starts_with("chore: land candidate A of run"),
19873 "{}",
19874 m.title
19875 );
19876 }
19877 }
19878
19879 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19880 BranchFacts {
19881 commits: commits
19882 .iter()
19883 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19884 .collect(),
19885 stat: stat.to_owned(),
19886 }
19887 }
19888
19889 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19890 let mut r = review_round(true, 0, 1, 1, true, true);
19891 r.round = round;
19892 r.fix = notes.map(|n| FixRecord {
19893 agent: "fixer".to_owned(),
19894 addressed: Vec::new(),
19895 rejected: Vec::new(),
19896 notes: n.to_owned(),
19897 committed: true,
19898 failed: None,
19899 duration_ms: 0,
19900 continuation: None,
19901 });
19902 r
19903 }
19904
19905 #[test]
19906 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19907 let state = review_state(&["ignored"]);
19908 let f = facts(
19909 &[
19910 (
19911 "fix(login): resolve PATH on macOS",
19912 "Login shells skip rc files.",
19913 ),
19914 ("fix: address review", ""),
19915 ],
19916 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19917 );
19918 let m = pr_message_with(&state, 'A', Some(&f));
19919 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19920 assert!(
19921 m.body
19922 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
19923 "{}",
19924 m.body
19925 );
19926 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19927 assert!(m.body.contains("## Diff stat"), "{}", m.body);
19928 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19929 for banned in [
19930 "Review the work already",
19931 "no task statement",
19932 "Original task",
19933 ] {
19934 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19935 }
19936 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19937 }
19938
19939 #[test]
19940 fn pr_message_review_truncates_a_huge_first_commit_body() {
19941 let state = review_state(&["ignored"]);
19942 let f = facts(
19943 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19944 "s",
19945 );
19946 let m = pr_message_with(&state, 'A', Some(&f));
19947 assert!(m.body.len() < 30_000, "{}", m.body.len());
19948 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19949 assert!(
19950 m.body.contains("- ... 1 more commit(s)"),
19951 "{}",
19952 m.body.len()
19953 );
19954 assert!(m.body.ends_with("magi:candidate-a\n"));
19955 }
19956
19957 #[test]
19958 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19959 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19960 assert_eq!(m.title, "feat: x");
19961 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19962 assert!(!m.body.contains("Diff stat"), "{}", m.body);
19963 assert!(!m.body.contains("no task statement"), "{}", m.body);
19964 }
19965
19966 #[test]
19967 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19968 let mut state = review_state(&["日本語の件名"]);
19969 state.candidates[0].branch = "fix/macos-login-path".to_owned();
19970 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19971 state.candidates[0].branch = "機能/ログイン".to_owned();
19972 assert!(
19973 pr_message(&state, 'A')
19974 .title
19975 .starts_with("chore: land candidate A")
19976 );
19977 }
19978
19979 #[test]
19980 fn pr_message_review_fixes_survive_a_clean_final_round() {
19981 let mut state = review_state(&["feat: x"]);
19982 state.reviews = vec![
19983 round_with_notes(1, Some("handled the PATH case")),
19984 round_with_notes(2, None),
19985 ];
19986 let body = pr_message(&state, 'A').body;
19987 assert!(
19988 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19989 "{body}"
19990 );
19991 assert!(body.contains("### Round 1"), "{body}");
19992
19993 state.reviews = vec![
19994 round_with_notes(1, Some("first fix")),
19995 round_with_notes(2, Some("")),
19996 round_with_notes(3, Some("second fix")),
19997 round_with_notes(4, None),
19998 ];
19999 let body = pr_message(&state, 'A').body;
20000 assert!(
20001 body.contains("### Round 1") && !body.contains("first fix"),
20002 "{body}"
20003 );
20004 assert!(
20005 body.contains("### Round 3") && !body.contains("second fix"),
20006 "{body}"
20007 );
20008 assert!(body.contains("### Round 2"), "{body}");
20009 }
20010
20011 const FINAL: &str = "abcdef1234567890";
20012
20013 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
20014 let mut r = round_with_notes(round, None);
20015 r.verified_head = verified.map(str::to_owned);
20016 r
20017 }
20018
20019 fn ok_gate(state: &mut RunState, head: Option<&str>) {
20020 state.gate = vec![CommandOutcome {
20021 command: "gate".to_owned(),
20022 code: Some(0),
20023 output_tail: String::new(),
20024 duration_ms: 0,
20025 resource_blocked: false,
20026 }];
20027 state.gate_ran = true;
20028 state.gate_head = head.map(str::to_owned);
20029 }
20030
20031 fn body_at(state: &RunState) -> String {
20032 pr_message_raw(
20033 state,
20034 'A',
20035 None,
20036 crate::pr_outcome::FinalHead::At(FINAL),
20037 ChangeNote::NotAsked,
20038 )
20039 .body
20040 }
20041
20042 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
20043
20044 #[test]
20045 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
20046 let mut state = review_state(&["feat: x"]);
20047 state.reviews = vec![
20048 round_with_notes(1, Some(SANDBOXED)),
20049 passing_round(2, Some(FINAL)),
20050 ];
20051 ok_gate(&mut state, Some(FINAL));
20052 let body = body_at(&state);
20053 assert!(!body.contains("Handled the PATH case."), "{body}");
20054 assert!(!body.contains("could not commit"), "{body}");
20055 assert!(!body.contains("could not run"), "{body}");
20056 assert!(
20057 body.contains("The fix is committed on the branch"),
20058 "{body}"
20059 );
20060 assert!(!body.contains("sandbox"), "{body}");
20061 assert!(
20062 body.contains("Tests (`verify.e2e`): passed on the final commit"),
20063 "{body}"
20064 );
20065 assert!(
20066 body.contains("Gate (`verify.gate`): passed on the final commit"),
20067 "{body}"
20068 );
20069 assert!(
20071 state.reviews[0]
20072 .fix
20073 .as_ref()
20074 .unwrap()
20075 .notes
20076 .contains("could not commit")
20077 );
20078 }
20079
20080 #[test]
20081 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
20082 let mut state = review_state(&["feat: x"]);
20083 let mut r1 = round_with_notes(1, Some(SANDBOXED));
20084 r1.fix.as_mut().unwrap().committed = false;
20085 let mut r2 = passing_round(2, Some(FINAL));
20086 r2.e2e[0].code = Some(1);
20087 state.reviews = vec![r1, r2];
20088 let body = body_at(&state);
20089 assert!(!body.contains("could not commit"), "{body}");
20090 assert!(!body.contains("could not run"), "{body}");
20091 assert!(body.contains("This round produced no commit"), "{body}");
20092 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
20093 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
20094 assert!(!body.contains("passed"), "{body}");
20095 }
20096
20097 #[test]
20098 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
20099 let mut state = review_state(&["feat: x"]);
20100 state.reviews = vec![
20101 round_with_notes(1, Some(SANDBOXED)),
20102 passing_round(2, Some("0123456789abcdef")),
20103 ];
20104 ok_gate(&mut state, Some("0123456789abcdef"));
20105 let body = body_at(&state);
20106 assert!(!body.contains("HTTP tests could not run"), "{body}");
20107 assert!(!body.contains("could not commit"), "{body}");
20109 assert!(
20110 body.contains("passed on an earlier commit (`0123456`)"),
20111 "{body}"
20112 );
20113 assert!(!body.contains("passed on the final commit"), "{body}");
20114
20115 ok_gate(&mut state, None);
20117 let body = body_at(&state);
20118 assert!(
20119 body.contains("passed on a commit that was not recorded"),
20120 "{body}"
20121 );
20122 }
20123
20124 #[test]
20125 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
20126 let state = state_with_summary(
20127 "add retries to the client",
20128 "TITLE: feat: retries\n\nDid it.",
20129 );
20130 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
20131 assert_eq!(m.title, "feat: retries");
20132 assert!(
20133 m.body.contains("<summary>Original task</summary>"),
20134 "{}",
20135 m.body
20136 );
20137 assert!(!m.body.contains("Commits under review"), "{}", m.body);
20138 assert!(
20139 m.body
20140 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
20141 "{}",
20142 m.body
20143 );
20144 }
20145
20146 #[test]
20147 fn pr_message_review_bounds_a_long_english_subject() {
20148 let long = format!("feat: {}", "word ".repeat(100));
20149 let m = pr_message(&review_state(&[&long]), 'A');
20150 assert!(m.title.starts_with("feat: word"), "{}", m.title);
20151 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20152 }
20153
20154 #[test]
20155 fn pr_message_implementation_run_is_unchanged_by_review_support() {
20156 let state = state_with_summary("add retries\n\ndetails", "- did some things");
20157 let m = pr_message(&state, 'A');
20158 assert_eq!(m.title, "add retries");
20159 assert!(m.body.contains("<summary>Original task</summary>"));
20160 assert!(!m.body.contains("Commits under review"));
20161 assert_eq!(landing_subject_source(&state), state.instruction);
20162 }
20163
20164 #[test]
20165 fn review_run_squash_subject_is_the_change_not_the_prompt() {
20166 let state = review_state(&["feat: the change", "fix: typo"]);
20167 let source = landing_subject_source(&state);
20168 assert_eq!(land::merge_subject("", &source), "feat: the change");
20169 assert_eq!(
20170 land::merge_subject("magi: candidate A (uncommitted work)", &source),
20171 "feat: the change"
20172 );
20173 assert_eq!(
20175 land::merge_subject("feat: renamed by hand", &source),
20176 "feat: renamed by hand"
20177 );
20178 let blank = review_state(&["日本語"]);
20179 assert!(
20180 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
20181 );
20182 }
20183
20184 #[test]
20185 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
20186 let state = review_state(&["feat: the change"]);
20187 let source = landing_subject_source(&state);
20188 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
20189 assert_eq!(
20190 land::merge_subject(landing_title(&state, old), &source),
20191 "feat: the change"
20192 );
20193 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
20194 let task = state_with_summary("add retries", "");
20195 assert_eq!(landing_title(&task, old), old);
20196 }
20197
20198 #[test]
20199 fn pr_message_describes_the_change_not_the_task() {
20200 let state = state_with_summary(
20201 "今回やってほしいこと: results projector を直す",
20202 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
20203 );
20204 let m = pr_message(&state, 'A');
20205 assert_eq!(m.title, "fix(web): batch the runs list reads");
20206 assert!(
20207 m.body.starts_with("## Summary\n\n- reads run.json once"),
20208 "{}",
20209 m.body
20210 );
20211 assert!(!m.body.contains("TITLE:"), "{}", m.body);
20212 let task_at = m.body.find("今回やってほしいこと").unwrap();
20213 let details_at = m.body.find("<details>").unwrap();
20214 assert!(
20215 details_at < task_at,
20216 "the task lives inside <details>: {}",
20217 m.body
20218 );
20219 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
20220 assert!(m.body.contains("magi:candidate-a"));
20221 }
20222
20223 #[test]
20224 fn pr_message_falls_back_to_the_task_without_a_title_line() {
20225 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
20226 let m = pr_message(&state, 'A');
20227 assert_eq!(m.title, "add retries");
20228 assert!(
20229 m.body.contains("## Summary\n\n- did some things"),
20230 "{}",
20231 m.body
20232 );
20233
20234 let none = RunState::new(
20235 PathBuf::from("/repo"),
20236 "main".to_owned(),
20237 "abc1234".to_owned(),
20238 "add retries".to_owned(),
20239 Config::default(),
20240 );
20241 let m = pr_message(&none, 'A');
20242 assert_eq!(m.title, "add retries");
20243 assert!(!m.body.contains("## Summary"), "{}", m.body);
20244 }
20245
20246 #[test]
20247 fn pr_message_refuses_the_candidate_commit_subject() {
20248 for bad in [
20249 "TITLE: magi: candidate A (uncommitted work)",
20250 "TITLE: chore: stuff (uncommitted work)",
20251 "TITLE: ",
20252 ] {
20253 let state = state_with_summary("add retries", bad);
20254 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
20255 }
20256 }
20257
20258 #[test]
20259 fn pr_message_bounds_a_very_long_task_and_title() {
20260 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
20261 let state = state_with_summary(&long, "- nothing");
20262 let m = pr_message(&state, 'A');
20263 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20264 assert!(!m.title.contains('\n'));
20265
20266 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
20267 let m = pr_message(&state, 'A');
20268 assert!(m.title.starts_with("feat: "));
20269 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
20270 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
20271 }
20272
20273 fn long_title_of(instruction: &str) -> String {
20274 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
20275 }
20276
20277 #[test]
20278 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
20279 let first = "Make the landing path keep a readable title for long tasks";
20280 let line = format!(
20281 "{first}. {}",
20282 "And then keep going with more words ".repeat(20)
20283 );
20284 let t = long_title_of(&line);
20285 assert_eq!(t, first);
20286 assert!(!t.starts_with("chore: land"));
20287 }
20288
20289 #[test]
20290 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
20291 let line = "word ".repeat(200);
20292 let t = long_title_of(&line);
20293 assert!(t.ends_with("word..."), "{t}");
20294 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
20295 }
20296
20297 #[test]
20298 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
20299 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
20300 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
20301 }
20302 }
20303
20304 #[test]
20305 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
20306 let mut state = state_with_summary("再試行を追加する", "- nothing");
20307 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
20308 state.config.graph.github_language = "ja".into();
20309 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
20310 }
20311
20312 #[test]
20313 fn pr_message_title_limit_is_exact() {
20314 let at = "a".repeat(PR_TITLE_MAX);
20315 assert_eq!(long_title_of(&at), at);
20316 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
20317 assert!(over.ends_with("..."), "{over}");
20318 assert_eq!(over.chars().count(), PR_TITLE_MAX);
20319 }
20320
20321 #[test]
20322 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
20323 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
20324 let t = long_title_of(&line);
20325 assert!(t.ends_with("..."), "{t}");
20326 assert!(t.is_ascii(), "{t}");
20327 }
20328
20329 #[test]
20330 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
20331 let line = format!(
20332 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
20333 "plus more filler words ".repeat(20)
20334 );
20335 let t = long_title_of(&line);
20336 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
20337 }
20338
20339 #[test]
20340 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
20341 let line = format!("\n\n# {}", "title words ".repeat(40));
20342 let state = state_with_summary(&line, "- nothing");
20343 let m = pr_message(&state, 'A');
20344 assert!(m.title.starts_with("title words"), "{}", m.title);
20345 assert_eq!(
20346 land::merge_subject(&m.title, &landing_subject_source(&state)),
20347 m.title
20348 );
20349 assert_eq!(
20351 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
20352 "feat: renamed by hand"
20353 );
20354 }
20355
20356 #[tokio::test]
20357 async fn github_text_rewrite_is_bounded_and_falls_back() {
20358 crate::run::pin_test_home();
20359 for (reply, accepted) in [
20360 (
20361 "TITLE: fix: retries\nAdd retries for failed requests.",
20362 true,
20363 ),
20364 (
20365 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
20366 false,
20367 ),
20368 (
20369 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
20370 true,
20371 ),
20372 (
20373 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
20374 true,
20375 ),
20376 (
20377 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
20378 true,
20379 ),
20380 (
20381 "TITLE: fix: retries\n日本語の説明です。 token=secret",
20382 false,
20383 ),
20384 ("TITLE: fix: retries\n", false),
20385 ] {
20386 let dir = tempfile::tempdir().unwrap();
20387 let mut runner = runner_at(RunStatus::Gating);
20388 runner.state = state_with_summary(
20389 "add retries",
20390 "TITLE: fix: retries\n日本語の説明を書きます。",
20391 );
20392 runner.state.repo = dir.path().to_owned();
20393 runner.state.candidates[0].worktree = dir.path().to_owned();
20394 let mut author = spec("alpha");
20395 author.command = vec![
20396 "sh".into(),
20397 "-c".into(),
20398 "printf '%s' \"$REWRITE_REPLY\"".into(),
20399 ];
20400 author.env.insert("REWRITE_REPLY".into(), reply.into());
20401 runner.state.config.agents = vec![author];
20402 seed_public_summary(&mut runner);
20403 let winner = runner.state.candidates[0].clone();
20404 let first = runner
20405 .guarded_pr_message(&winner, None, true)
20406 .await
20407 .unwrap();
20408 let message = if accepted {
20409 first.expect("a passing rewrite never asks")
20410 } else {
20411 assert!(first.is_none());
20413 assert!(runner.state.parked);
20414 let store = ask::Questions::open();
20415 let asked: Vec<_> = store
20416 .list()
20417 .into_iter()
20418 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20419 .collect();
20420 assert_eq!(asked.len(), 1);
20421 assert!(!asked[0].detail.contains("secret"));
20422 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
20423 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
20424 assert_eq!(
20425 asked[0]
20426 .choices
20427 .contains(&crate::github_text::USE_REWRITE.to_owned()),
20428 rewrite.is_some()
20429 );
20430 store
20433 .update(&asked[0].id, |q| {
20434 q.abandon("test");
20435 Ok(())
20436 })
20437 .unwrap();
20438 let again = runner
20439 .guarded_pr_message(&winner, None, true)
20440 .await
20441 .unwrap()
20442 .expect("an abandoned question falls back");
20443 let n = store
20444 .list()
20445 .into_iter()
20446 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20447 .count();
20448 assert_eq!(n, 1, "asked once per rejected text");
20449 again
20450 };
20451 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
20452 assert!(!message.body.contains("secret"));
20453 assert!(!message.body.contains("prefix"));
20454 assert!(!message.title.contains("secret"));
20455 assert_eq!(
20456 message.body.contains("Add retries for failed requests."),
20457 accepted
20458 );
20459 assert_eq!(
20460 runner.state.seats["github-text-rewrite"].turns, 1,
20461 "only one rewrite invocation"
20462 );
20463 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
20464 if accepted {
20465 assert!(runner.state.github_text.is_none());
20466 assert!(!runner.state.parked);
20467 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
20468 assert!(message.body.contains("## Final verification"));
20469 assert!(message.body.contains("Evidence: final diff"));
20470 assert!(
20471 message
20472 .body
20473 .contains(&format!("magi:run/{}", runner.state.id))
20474 );
20475 }
20476 if !accepted {
20477 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20478 assert!(
20479 message
20480 .body
20481 .contains(&format!("magi:run/{}", runner.state.id))
20482 );
20483 }
20484 }
20485 }
20486
20487 fn scripted(id: &str, reply: &str) -> AgentSpec {
20488 let mut a = spec(id);
20489 a.command = vec![
20490 "sh".into(),
20491 "-c".into(),
20492 "printf '%s' \"$REWRITE_REPLY\"".into(),
20493 ];
20494 a.env.insert("REWRITE_REPLY".into(), reply.into());
20495 a
20496 }
20497
20498 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
20499 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
20500
20501 fn seed_public_summary(runner: &mut Runner) {
20503 let repo = &runner.state.repo;
20504 let g = |args: &[&str]| {
20505 let out = std::process::Command::new("git")
20506 .args(args)
20507 .current_dir(repo)
20508 .quiet()
20509 .output()
20510 .unwrap();
20511 assert!(out.status.success(), "git {args:?}");
20512 String::from_utf8_lossy(&out.stdout).trim().to_owned()
20513 };
20514 g(&["init", "-q", "-b", "main"]);
20515 g(&[
20516 "-c",
20517 "user.name=Test",
20518 "-c",
20519 "user.email=test@example.invalid",
20520 "commit",
20521 "-q",
20522 "--allow-empty",
20523 "-m",
20524 "feat: fixture",
20525 ]);
20526 let head = g(&["rev-parse", "HEAD"]);
20527 g(&["update-ref", "refs/remotes/origin/main", &head]);
20528 let evidence = crate::pr_outcome::summary_evidence(
20529 &runner.state,
20530 crate::pr_outcome::FinalHead::At(&head),
20531 );
20532 let generated = &runner.state.candidates[0].summary;
20533 runner.state.change_summary = Some(crate::run::ChangeSummary {
20534 base: head.clone(),
20535 head,
20536 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
20537 title: summary_title(generated).unwrap(),
20538 body: summary_without_title(generated),
20539 });
20540 }
20541
20542 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
20543 let mut runner = runner_at(RunStatus::Gating);
20544 runner.state = state_with_summary("add retries", JA_SUMMARY);
20545 if review_only {
20546 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
20547 }
20548 runner.state.repo = dir.to_owned();
20549 runner.state.candidates[0].worktree = dir.to_owned();
20550 runner.state.config.agents = agents;
20551 seed_public_summary(&mut runner);
20552 runner
20553 }
20554
20555 fn gate_questions(runner: &Runner) -> usize {
20556 ask::Questions::open()
20557 .list()
20558 .into_iter()
20559 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
20560 .count()
20561 }
20562
20563 #[tokio::test]
20564 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
20565 crate::run::pin_test_home();
20566 let dir = tempfile::tempdir().unwrap();
20567 let mut runner = rewrite_runner(
20568 dir.path(),
20569 true,
20570 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20571 );
20572 runner.state.config.roles.language_judge =
20573 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
20574 let winner = runner.state.candidates[0].clone();
20575 let msg = runner
20576 .guarded_pr_message(&winner, None, true)
20577 .await
20578 .unwrap()
20579 .expect("an English rewrite posts without asking");
20580 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
20581 assert!(msg.body.contains("<details>kept</details>"));
20582 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
20583 assert_eq!(gate_questions(&runner), 0);
20584 assert!(runner.state.github_text.is_none());
20585 assert!(!runner.state.seats.contains_key("impl-A"));
20586 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20587 }
20588
20589 #[tokio::test]
20590 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
20591 crate::run::pin_test_home();
20592 let dir = tempfile::tempdir().unwrap();
20593 let mut runner = rewrite_runner(
20594 dir.path(),
20595 false,
20596 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
20597 );
20598 runner.state.config.roles.language_judge =
20599 Some(crate::config::AgentChoice::One("beta".into()));
20600 runner.state.seats.insert(
20601 "impl-A".into(),
20602 SeatState::new("impl-A", "alpha", runner.state.seed),
20603 );
20604 let winner = runner.state.candidates[0].clone();
20605 let msg = runner
20606 .guarded_pr_message(&winner, None, true)
20607 .await
20608 .unwrap()
20609 .expect("rewritten");
20610 assert!(msg.body.contains("<details>kept</details>"));
20611 assert_eq!(runner.state.seats["impl-A"].turns, 0);
20612 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20613 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20614 }
20615
20616 #[tokio::test]
20617 async fn rewrite_reuses_a_resumable_implementer_session() {
20618 crate::run::pin_test_home();
20619 let dir = tempfile::tempdir().unwrap();
20620 let mut runner = rewrite_runner(
20621 dir.path(),
20622 false,
20623 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
20624 );
20625 runner.state.config.roles.language_judge =
20626 Some(crate::config::AgentChoice::One("beta".into()));
20627 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20628 seat.turns = 1;
20629 runner.state.seats.insert("impl-A".into(), seat);
20630 let winner = runner.state.candidates[0].clone();
20631 runner
20632 .guarded_pr_message(&winner, None, true)
20633 .await
20634 .unwrap()
20635 .expect("rewritten");
20636 assert_eq!(runner.state.seats["impl-A"].turns, 2);
20637 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
20638 }
20639
20640 #[tokio::test]
20641 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
20642 crate::run::pin_test_home();
20645 let dir = tempfile::tempdir().unwrap();
20646 let mut exhausted = spec("alpha");
20647 exhausted.command = vec!["false".into()];
20648 let mut runner = rewrite_runner(
20649 dir.path(),
20650 false,
20651 vec![exhausted, scripted("beta", EN_REWRITE)],
20652 );
20653 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20655 "alpha".into(),
20656 "beta".into(),
20657 ]));
20658 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
20659 seat.turns = 1;
20660 runner.state.seats.insert("impl-A".into(), seat);
20661 let winner = runner.state.candidates[0].clone();
20662 let msg = runner
20663 .guarded_pr_message(&winner, None, true)
20664 .await
20665 .unwrap()
20666 .expect("the role rewrites after the implementer fails");
20667 assert!(msg.body.contains("`src/net.rs`"));
20668 assert_eq!(
20669 runner.state.seats["impl-A"].turns, 2,
20670 "the implementer was tried once"
20671 );
20672 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
20673 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20674 }
20675
20676 #[tokio::test]
20677 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
20678 crate::run::pin_test_home();
20679 let dir = tempfile::tempdir().unwrap();
20680 let mut broken = spec("bad");
20681 broken.command = vec!["false".into()];
20682 let mut runner = rewrite_runner(
20683 dir.path(),
20684 true,
20685 vec![
20686 broken,
20687 scripted("noformat", "Add retries."),
20688 scripted("good", EN_REWRITE),
20689 ],
20690 );
20691 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
20692 "bad".into(),
20693 "noformat".into(),
20694 "good".into(),
20695 ]));
20696 let winner = runner.state.candidates[0].clone();
20697 let msg = runner
20698 .guarded_pr_message(&winner, None, true)
20699 .await
20700 .unwrap()
20701 .expect("the third agent rewrites");
20702 assert!(msg.body.contains("`src/net.rs`"));
20703 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
20704 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
20705 }
20706
20707 #[tokio::test]
20708 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
20709 crate::run::pin_test_home();
20710 for (agents, role, saved) in [
20712 (
20713 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
20714 Some("beta"),
20715 true,
20716 ),
20717 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
20718 (Vec::new(), None, false),
20719 ] {
20720 let dir = tempfile::tempdir().unwrap();
20721 let mut runner = rewrite_runner(dir.path(), true, agents);
20722 runner.state.config.roles.language_judge =
20723 role.map(|r| crate::config::AgentChoice::One(r.into()));
20724 let winner = runner.state.candidates[0].clone();
20725 let first = runner
20726 .guarded_pr_message(&winner, None, true)
20727 .await
20728 .unwrap();
20729 assert!(first.is_none(), "withheld text parks the run");
20730 assert!(runner.state.parked);
20731 assert_eq!(gate_questions(&runner), 1);
20732 let g = runner.state.github_text.as_ref().unwrap();
20733 assert_eq!(g.rewrite.is_some(), saved);
20734 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20735 }
20736 }
20737
20738 #[tokio::test]
20739 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20740 crate::run::pin_test_home();
20741 let dir = tempfile::tempdir().unwrap();
20742 let mut runner = runner_at(RunStatus::Gating);
20743 runner.state =
20744 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20745 runner.state.repo = dir.path().to_owned();
20746 runner.state.candidates[0].worktree = dir.path().to_owned();
20747 let mut author = spec("alpha");
20748 author.command = vec![
20749 "sh".into(),
20750 "-c".into(),
20751 "printf '%s' \"$REWRITE_REPLY\"".into(),
20752 ];
20753 author.env.insert(
20754 "REWRITE_REPLY".into(),
20755 format!(
20756 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20757 "English sentence. ".repeat(600)
20758 ),
20759 );
20760 runner.state.config.agents = vec![author];
20761 seed_public_summary(&mut runner);
20762 let winner = runner.state.candidates[0].clone();
20763 assert!(
20764 runner
20765 .guarded_pr_message(&winner, None, true)
20766 .await
20767 .unwrap()
20768 .is_none()
20769 );
20770 let record = runner.state.github_text.clone().unwrap();
20771 let rewrite = record.rewrite.clone().unwrap();
20772 assert_eq!(rewrite.failed, ["body-language"]);
20773 assert!(
20774 rewrite
20775 .body
20776 .ends_with(&format!("magi:run/{}", runner.state.id))
20777 );
20778 let store = ask::Questions::open();
20779 let q = store.get(record.question.as_ref().unwrap()).unwrap();
20780 assert!(
20781 q.choices
20782 .contains(&crate::github_text::USE_REWRITE.to_owned())
20783 );
20784 for label in [
20785 "ORIGINAL",
20786 "FALLBACK",
20787 "REWRITE",
20788 "Rules still failed: body-language",
20789 "truncated",
20790 ] {
20791 assert!(q.detail.contains(label), "{label}");
20792 }
20793 let artifact = std::fs::read_to_string(crate::run::artifact_path(
20794 &runner.state,
20795 &crate::github_text::artifact_name(&record.fingerprint),
20796 ))
20797 .unwrap();
20798 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20799 runner.state =
20801 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20802 store
20803 .update(&q.id, |q| {
20804 q.say("まだ日本語")?;
20805 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20806 })
20807 .unwrap();
20808 assert!(
20809 runner
20810 .guarded_pr_message(&winner, None, true)
20811 .await
20812 .unwrap()
20813 .is_none()
20814 );
20815 let retry = runner.state.github_text.as_ref().unwrap();
20816 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20817 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20818 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20819 assert!(
20820 q.posting_comparison
20821 .as_ref()
20822 .unwrap()
20823 .matches_detail(&q.detail)
20824 );
20825 store
20826 .update(&q.id, |q| {
20827 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20828 })
20829 .unwrap();
20830 let message = runner
20831 .guarded_pr_message(&winner, None, true)
20832 .await
20833 .unwrap()
20834 .unwrap();
20835 assert_eq!(message.title, rewrite.title);
20836 assert_eq!(message.body, rewrite.body);
20837 assert_eq!(
20838 runner.state.github_text.as_ref().unwrap().choice,
20839 Some(GithubTextChoice::Rewrite)
20840 );
20841 let again = runner
20842 .guarded_pr_message(&winner, None, true)
20843 .await
20844 .unwrap()
20845 .unwrap();
20846 assert_eq!(again.title, message.title);
20847 assert_eq!(again.body, message.body);
20848 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20849 assert!(
20850 !runner
20851 .state
20852 .events
20853 .iter()
20854 .any(|e| e.message.contains(&rewrite.title)
20855 || e.message.contains("English sentence."))
20856 );
20857 }
20858
20859 #[tokio::test]
20860 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20861 crate::run::pin_test_home();
20862 let dir = tempfile::tempdir().unwrap();
20863 let mut runner = runner_at(RunStatus::Gating);
20864 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20865 runner.state.repo = dir.path().to_owned();
20866 runner.state.candidates[0].worktree = dir.path().to_owned();
20867 seed_public_summary(&mut runner);
20868 let winner = runner.state.candidates[0].clone();
20869 assert!(
20870 runner
20871 .guarded_pr_message(&winner, None, true)
20872 .await
20873 .unwrap()
20874 .is_none()
20875 );
20876 let record = runner.state.github_text.as_ref().unwrap();
20877 assert!(record.rewrite.is_none());
20878 let mut old = serde_json::to_value(record).unwrap();
20880 old.as_object_mut().unwrap().remove("rewrite");
20881 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20882 assert!(old.rewrite.is_none());
20883 let id = record.question.clone().unwrap();
20884 ask::Questions::open()
20885 .update(&id, |q| {
20886 q.choices.push(crate::github_text::USE_REWRITE.into());
20887 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20888 })
20889 .unwrap();
20890 let message = runner
20891 .guarded_pr_message(&winner, None, true)
20892 .await
20893 .unwrap()
20894 .unwrap();
20895 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20896 assert_eq!(
20897 runner.state.github_text.as_ref().unwrap().choice,
20898 Some(GithubTextChoice::Fallback)
20899 );
20900 }
20901
20902 #[tokio::test]
20903 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20904 crate::run::pin_test_home();
20905 let dir = tempfile::tempdir().unwrap();
20906 let mut runner = runner_at(RunStatus::Gating);
20907 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20908 runner.state.repo = dir.path().to_owned();
20909 runner.state.candidates[0].worktree = dir.path().to_owned();
20910 seed_public_summary(&mut runner);
20911 let winner = runner.state.candidates[0].clone();
20912 assert!(
20913 runner
20914 .guarded_pr_message(&winner, None, true)
20915 .await
20916 .unwrap()
20917 .is_none()
20918 );
20919 let store = ask::Questions::open();
20920 let find = |run: &str| {
20921 store
20922 .list()
20923 .into_iter()
20924 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20925 .unwrap()
20926 };
20927 let run = runner.state.id.clone();
20928 let q = find(&run);
20929 let fp = runner
20930 .state
20931 .github_text
20932 .as_ref()
20933 .unwrap()
20934 .fingerprint
20935 .clone();
20936 let name = crate::github_text::artifact_name(&fp);
20937 let path = crate::run::artifact_path(&runner.state, &name);
20938 assert!(q.detail.contains(&path.display().to_string()));
20939 assert!(q.detail.contains("title-language: title"));
20940 let saved = std::fs::read_to_string(&path).expect("artifact written");
20941 assert!(saved.contains("日本語のタイトル"));
20942 assert!(q.cwd.is_none());
20943 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20944 let say = |id: &str, text: &str, choice: &str| {
20946 store
20947 .update(id, |q| {
20948 q.thread.push(ask::Turn {
20949 who: ask::Who::Operator,
20950 body: text.to_owned(),
20951 at: jiff::Timestamp::now(),
20952 note: None,
20953 });
20954 q.answer(ask::Answer::Choice(choice.to_owned()))
20955 })
20956 .unwrap();
20957 };
20958 say(&q.id, "まだ日本語", "use my text");
20959 assert!(
20960 runner
20961 .guarded_pr_message(&winner, None, true)
20962 .await
20963 .unwrap()
20964 .is_none()
20965 );
20966 let second = find(&run);
20967 assert_ne!(second.id, q.id);
20968 assert!(second.detail.contains("did not pass"));
20969 assert!(second.detail.contains("日本語のタイトル"));
20972 assert!(!second.detail.contains("まだ日本語"));
20973 assert!(second.detail.contains(&path.display().to_string()));
20974 say(&second.id, "fix: retry failed requests", "use my text");
20975 let message = runner
20976 .guarded_pr_message(&winner, None, true)
20977 .await
20978 .unwrap()
20979 .expect("a vetted replacement is posted");
20980 assert_eq!(message.title, "fix: retry failed requests");
20981 assert_eq!(
20982 runner
20983 .state
20984 .github_text
20985 .as_ref()
20986 .unwrap()
20987 .chosen_title
20988 .as_deref(),
20989 Some("fix: retry failed requests")
20990 );
20991 assert!(runner.state.github_text.as_ref().unwrap().resolved);
20992 let again = runner
20994 .guarded_pr_message(&winner, None, true)
20995 .await
20996 .unwrap()
20997 .unwrap();
20998 assert_eq!(again.title, message.title);
20999 assert_eq!(
21000 store
21001 .list()
21002 .into_iter()
21003 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
21004 .count(),
21005 2
21006 );
21007 }
21008
21009 #[test]
21010 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
21011 let mut state = state_with_summary(
21015 "add retries",
21016 "TITLE: fix(web): batch reads\n- reads run.json once",
21017 );
21018 state.config.graph.language = "ja".to_owned();
21019 let m = pr_message(&state, 'A');
21020 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
21021 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
21022
21023 let task = "今回やってほしいこと: results projector を直す";
21026 let mut state = state_with_summary(task, "- no title line");
21027 state.config.graph.language = "ja".to_owned();
21028 let m = pr_message(&state, 'A');
21029 assert_eq!(
21030 m.title,
21031 format!("chore: land candidate A of run {}", state.id)
21032 );
21033 assert!(
21034 m.body.contains(&format!(
21035 "<summary>Original task</summary>\n\n{task}\n\n</details>"
21036 )),
21037 "{}",
21038 m.body
21039 );
21040 }
21041
21042 #[test]
21043 fn pr_message_scrubs_home_paths_and_addresses() {
21044 let state = state_with_summary(
21045 "fix it in /Users/someone/src/x",
21046 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
21047 );
21048 let m = pr_message(&state, 'A');
21049 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
21050 assert!(!m.body.contains(leak), "{}", m.body);
21051 }
21052 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
21053 }
21054
21055 #[test]
21056 fn pr_message_survives_a_task_that_closes_details() {
21057 let state = state_with_summary("a </details> b", "TITLE: fix: x");
21058 let m = pr_message(&state, 'A');
21059 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
21060 }
21061
21062 #[test]
21063 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
21064 let cmd = manual_merge_command(
21065 MergeStyle::Squash,
21066 Path::new("/repo"),
21067 "b",
21068 "fix: \"quoted\" $(x) `y`\n\nbody",
21069 );
21070 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
21071 }
21072
21073 #[test]
21074 fn manual_merge_command_matches_the_configured_style() {
21075 let repo = Path::new("/repo");
21076 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
21077
21078 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
21079 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
21080
21081 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
21082 assert_eq!(
21083 squash,
21084 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
21085 \"Merge magi run 0832 (candidate A)\""
21086 );
21087
21088 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
21089 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
21090 }
21091
21092 #[test]
21093 fn a_nudge_gets_a_quarter_of_the_budget() {
21094 assert_eq!(retry_budget(secs(1200), true), secs(300));
21096 assert_eq!(retry_budget(secs(3600), true), secs(900));
21097 }
21098
21099 #[test]
21100 fn a_resent_prompt_keeps_the_whole_budget() {
21101 assert_eq!(retry_budget(secs(1200), false), secs(1200));
21104 assert_eq!(retry_budget(secs(60), false), secs(60));
21105 }
21106
21107 #[test]
21108 fn the_floor_never_exceeds_the_original_budget() {
21109 assert_eq!(retry_budget(secs(60), true), secs(60));
21113 assert_eq!(retry_budget(secs(480), true), secs(120));
21114 assert_eq!(retry_budget(secs(0), true), secs(0));
21115 }
21116
21117 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
21118 agent::CommandEvidence {
21119 id: "item1".to_owned(),
21120 description: "cargo test".to_owned(),
21121 exit_code,
21122 result_summary: String::new(),
21123 source: "codex".to_owned(),
21124 }
21125 }
21126
21127 #[test]
21128 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
21129 assert!(!has_unconfirmed_command(&[]));
21133 }
21134
21135 #[test]
21136 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
21137 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
21141 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
21142 assert!(!has_unconfirmed_command(&[
21143 evidence(Some(0)),
21144 evidence(Some(101))
21145 ]));
21146 }
21147
21148 #[test]
21149 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
21150 assert!(has_unconfirmed_command(&[
21151 evidence(Some(0)),
21152 evidence(None)
21153 ]));
21154 }
21155
21156 #[test]
21157 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
21158 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21159 assert_eq!(
21160 verified_noop_claim(true, &[], text).as_deref(),
21161 Some("already fixed by b32cfc4, on main.")
21162 );
21163 }
21164
21165 #[test]
21166 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
21167 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21170 assert!(verified_noop_claim(false, &[], text).is_none());
21171 }
21172
21173 #[test]
21174 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
21175 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
21176 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
21177 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
21179 }
21180
21181 #[test]
21182 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
21183 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
21184 }
21185
21186 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
21189 runner.state.candidates = shape
21190 .iter()
21191 .enumerate()
21192 .map(|(i, &(empty, verified))| Candidate {
21193 index: i,
21194 label: (b'A' + i as u8) as char,
21195 agent: "sonnet".to_owned(),
21196 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
21197 worktree: PathBuf::from(format!("/wt/{i}")),
21198 summary: String::new(),
21199 stat: String::new(),
21200 files: 0,
21201 commits: 0,
21202 empty,
21203 failed: None,
21204 verified_noop: verified.map(str::to_owned),
21205 duration_ms: 0,
21206 folded: false,
21207 })
21208 .collect();
21209 }
21210
21211 #[test]
21212 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
21213 ask_test_home();
21214 let mut runner = runner_at(RunStatus::Implementing);
21215 set_candidates(
21216 &mut runner,
21217 &[
21218 (true, Some("already on main at b32cfc4")),
21219 (true, Some("same fix, see the existing test")),
21220 ],
21221 );
21222
21223 runner
21224 .after_implement()
21225 .expect("a verified no-op is not an error");
21226
21227 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
21228 }
21229
21230 #[test]
21231 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
21232 ask_test_home();
21233 let mut runner = runner_at(RunStatus::Implementing);
21234 set_candidates(
21238 &mut runner,
21239 &[(true, Some("already on main at b32cfc4")), (true, None)],
21240 );
21241
21242 let err = runner
21243 .after_implement()
21244 .expect_err("an unverified empty candidate must still fail the run");
21245
21246 assert!(
21247 err.to_string().contains("no candidate produced a change"),
21248 "{err}"
21249 );
21250 assert_eq!(runner.state.status, RunStatus::Failed);
21251 }
21252
21253 #[test]
21254 fn after_implement_still_fails_an_ordinary_all_empty_run() {
21255 ask_test_home();
21256 let mut runner = runner_at(RunStatus::Implementing);
21257 set_candidates(&mut runner, &[(true, None), (true, None)]);
21258
21259 let err = runner
21260 .after_implement()
21261 .expect_err("no candidate declared anything; this is an ordinary failure");
21262
21263 assert!(
21264 err.to_string().contains("no candidate produced a change"),
21265 "{err}"
21266 );
21267 assert_eq!(runner.state.status, RunStatus::Failed);
21268 }
21269}
21270
21271#[derive(Debug)]
21274struct GraftRestoreFailed(String);
21275
21276impl std::fmt::Display for GraftRestoreFailed {
21277 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
21278 write!(
21279 f,
21280 "could not restore the winner to its pre-graft commit: {}",
21281 self.0
21282 )
21283 }
21284}
21285
21286impl std::error::Error for GraftRestoreFailed {}
21287
21288#[cfg(test)]
21292mod resource_wait_tests {
21293 use super::*;
21294 use std::collections::BTreeMap;
21295
21296 fn sh(id: &str, script: &str) -> AgentSpec {
21297 AgentSpec {
21298 id: id.to_owned(),
21299 kind: crate::config::AgentKind::Command,
21300 model: None,
21301 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
21302 extra_args: Vec::new(),
21303 env: BTreeMap::new(),
21304 prompt_delivery: None,
21305 max_turn_secs: None,
21306 }
21307 }
21308
21309 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
21310 SeatJob {
21311 seat: SeatState::new("impl-A", &spec.id, 1),
21312 spec,
21313 cwd: dir.to_path_buf(),
21314 prompt: "do it".to_owned(),
21315 timeout,
21316 allow_write: true,
21317 sessions: false,
21318 artifacts: dir.join("art"),
21319 stem: "impl-A".to_owned(),
21320 handover: None,
21321 }
21322 }
21323
21324 fn state(resource_wait_secs: u64) -> RunState {
21325 let mut config = Config::default();
21326 config.graph.timeout_resource_wait = resource_wait_secs;
21327 RunState::new(
21328 PathBuf::from("/repo"),
21329 "main".to_owned(),
21330 "abc1234".to_owned(),
21331 "task".to_owned(),
21332 config,
21333 )
21334 }
21335
21336 fn hold(cache: &Path) -> crate::cache::Guard {
21337 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
21338 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
21339 crate::cache::AcquireOutcome::Acquired(g) => g,
21340 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
21341 }
21342 }
21343
21344 async fn run(
21345 jobs: Vec<SeatJob>,
21346 sem: Arc<Semaphore>,
21347 cache: Option<&Path>,
21348 st: &mut RunState,
21349 ) -> Vec<(usize, SeatState, AgentOutcome)> {
21350 let prompts = Prompts::default();
21351 let ctx = WaveCtx {
21352 run: "r",
21353 node: "implement",
21354 prompts: &prompts,
21355 cache,
21356 round: None,
21357 carry_seats: false,
21358 pauses: None,
21359 };
21360 wave(jobs, sem, &ctx, st, 0).await
21361 }
21362
21363 #[tokio::test]
21364 async fn verification_cache_wait_preserves_the_command_budget() {
21365 crate::run::pin_test_home();
21366 let tmp = tempfile::tempdir().unwrap();
21367 let cache = tmp.path().join("target");
21368 let guard = hold(&cache);
21369 let release = tokio::spawn(async move {
21370 tokio::time::sleep(Duration::from_millis(100)).await;
21371 guard.release();
21372 });
21373 let mut st = state(10);
21374 let budget = Duration::from_millis(50);
21375 let wait = repository_wait(&st, None);
21376 let (outcomes, _) = with_cache_lease(
21377 &mut st,
21378 Some(&cache),
21379 "gate",
21380 "gate",
21381 tmp.path(),
21382 "abc1234",
21383 budget,
21384 "test gate",
21385 &wait,
21386 |_, available| async move {
21387 assert_eq!(
21388 available, budget,
21389 "contention must not shorten verification"
21390 );
21391 (Vec::new(), false, Vec::new())
21392 },
21393 )
21394 .await
21395 .unwrap();
21396 release.await.unwrap();
21397 assert!(outcomes.is_empty(), "{outcomes:?}");
21398 assert!(
21399 st.events
21400 .iter()
21401 .any(|event| event.message.contains("waiting for"))
21402 );
21403 assert!(st.resource_wait.is_none());
21404 hold(&cache).release();
21406 }
21407
21408 #[tokio::test]
21409 async fn verification_cache_wait_expires_without_starting_commands() {
21410 crate::run::pin_test_home();
21411 let tmp = tempfile::tempdir().unwrap();
21412 let cache = tmp.path().join("target");
21413 let _held = hold(&cache);
21414 let mut st = state(0);
21415 let wait = repository_wait(&st, None);
21416 let (outcomes, _) = with_cache_lease(
21417 &mut st,
21418 Some(&cache),
21419 "gate",
21420 "gate",
21421 tmp.path(),
21422 "abc1234",
21423 Duration::from_secs(60),
21424 "test gate",
21425 &wait,
21426 |_, _| async { panic!("commands must not start without the cache lease") },
21427 )
21428 .await
21429 .unwrap();
21430 assert_eq!(outcomes.len(), 1);
21431 assert!(outcomes[0].resource_blocked);
21432 assert!(st.resource_wait.is_none());
21433 assert!(matches!(
21434 st.resource_block.unwrap().cause,
21435 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
21436 ));
21437 }
21438
21439 #[tokio::test]
21440 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
21441 crate::run::pin_test_home();
21442 let tmp = tempfile::tempdir().unwrap();
21443 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
21444 let cache = tmp.path().join("cache");
21445 let mut state = state(21600);
21446 let wait = repository_wait(&state, None);
21447 let (outcomes, _) = with_cache_lease(
21448 &mut state,
21449 Some(&cache),
21450 "gate",
21451 "gate",
21452 tmp.path(),
21453 "head",
21454 Duration::from_secs(60),
21455 "gate",
21456 &wait,
21457 |_, _| async { panic!("freshness failure must not start verification") },
21458 )
21459 .await
21460 .unwrap();
21461 assert_eq!(outcomes.len(), 1);
21462 assert!(outcomes[0].resource_blocked);
21463 let block = state.resource_block.unwrap();
21464 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
21465 assert!(!block.description().contains("timed out"));
21466 }
21467
21468 #[tokio::test]
21469 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
21470 crate::run::pin_test_home();
21471 let tmp = tempfile::tempdir().unwrap();
21472 let cache = tmp.path().join("cache");
21473 let _held = hold(&cache);
21474 let mut state = state(21600);
21475 let pause = Pause::new();
21476 let interrupt = Pause::new();
21477 let cause = ParkCause::Operator;
21478 pause.park_for(cause.clone());
21479 let prompts = Prompts::default();
21480 let ctx = WaveCtx {
21481 run: "test",
21482 node: "fix",
21483 prompts: &prompts,
21484 cache: Some(&cache),
21485 round: None,
21486 carry_seats: false,
21487 pauses: Some((&pause, &interrupt)),
21488 };
21489 let result = wave(
21490 vec![job(
21491 sh("writer", "exit 99"),
21492 tmp.path(),
21493 Duration::from_secs(1),
21494 )],
21495 Arc::new(Semaphore::new(1)),
21496 &ctx,
21497 &mut state,
21498 0,
21499 )
21500 .await;
21501 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
21502 assert_eq!(result[0].1.turns, 0);
21503 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21504 assert_eq!(
21505 state.resource_block.unwrap().cause,
21506 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
21507 );
21508 assert!(state.active.is_empty());
21509 }
21510
21511 #[tokio::test]
21512 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
21513 crate::run::pin_test_home();
21514 let tmp = tempfile::tempdir().unwrap();
21515 let cache = tmp.path().join("target-a");
21516 let guard = hold(&cache);
21517 tokio::spawn(async move {
21519 tokio::time::sleep(Duration::from_millis(1500)).await;
21520 guard.release();
21521 });
21522 let mut st = state(60);
21523 let results = run(
21524 vec![job(
21525 sh("alpha", "echo ok"),
21526 tmp.path(),
21527 Duration::from_secs(1),
21528 )],
21529 Arc::new(Semaphore::new(1)),
21530 Some(&cache),
21531 &mut st,
21532 )
21533 .await;
21534 assert!(
21535 matches!(results[0].2, AgentOutcome::Ok(_)),
21536 "the wait must not be charged to the agent"
21537 );
21538 assert!(st.resource_block.is_none());
21539 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21540 assert!(st.resource_wait.is_none() && st.active.is_empty());
21541 }
21542
21543 #[tokio::test]
21544 #[cfg(unix)]
21545 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
21546 use std::os::fd::AsRawFd as _;
21547 crate::run::pin_test_home();
21548 let tmp = tempfile::tempdir().unwrap();
21549 crate::git::git(tmp.path(), &["init"]).await.unwrap();
21550 let common = tmp.path().join(".git");
21551 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
21552 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
21553 assert_eq!(
21554 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
21555 0
21556 );
21557 let mut st = state(1);
21558 st.config.coordination = crate::config::test_coordination();
21559 let mut reader = job(
21560 sh("reader", "echo must-not-run"),
21561 tmp.path(),
21562 Duration::from_secs(30),
21563 );
21564 reader.allow_write = false;
21565 let results = run(
21566 vec![reader.clone()],
21567 Arc::new(Semaphore::new(1)),
21568 None,
21569 &mut st,
21570 )
21571 .await;
21572 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21573 assert_eq!(results[0].1.turns, 0);
21574 assert!(FailClass::of(&results[0].2).is_none());
21575 assert_eq!(
21576 st.resource_block.as_ref().unwrap().kind,
21577 "repository coordination"
21578 );
21579 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21580 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21581 drop(external);
21582 let editor = crate::repo_lock::SeatLease::for_checkout(
21583 tmp.path(),
21584 true,
21585 crate::config::test_coordination().as_ref(),
21586 )
21587 .await
21588 .unwrap();
21589 let pause = Pause::new();
21590 let interrupt = Pause::new();
21591 let prompts = Prompts::default();
21592 let ctx = WaveCtx {
21593 run: "r",
21594 node: "review",
21595 prompts: &prompts,
21596 cache: None,
21597 round: None,
21598 carry_seats: false,
21599 pauses: Some((&pause, &interrupt)),
21600 };
21601 let (results, ()) = tokio::join!(
21602 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
21603 async {
21604 tokio::time::sleep(Duration::from_millis(50)).await;
21605 pause.park();
21606 }
21607 );
21608 assert!(
21609 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
21610 );
21611 assert_eq!(results[0].1.turns, 0);
21612 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
21613 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
21614 drop(editor);
21615 }
21616
21617 #[tokio::test]
21618 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
21619 crate::run::pin_test_home();
21620 let tmp = tempfile::tempdir().unwrap();
21621 let cache = tmp.path().join("target-b");
21622 let _held = hold(&cache);
21623 let mut st = state(1);
21624 let results = run(
21625 vec![job(
21626 sh("alpha", "echo ok"),
21627 tmp.path(),
21628 Duration::from_secs(1),
21629 )],
21630 Arc::new(Semaphore::new(1)),
21631 Some(&cache),
21632 &mut st,
21633 )
21634 .await;
21635 let (_, seat, out) = &results[0];
21636 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
21637 assert!(wave_blocked(&results));
21638 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
21639 assert_eq!(seat.turns, 0, "nothing was asked");
21640 assert!(
21641 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
21642 "no agent was spawned"
21643 );
21644 assert!(FailClass::of(out).is_none(), "not a failure class");
21645 let block = st.resource_block.as_ref().expect("recorded for the resume");
21646 assert_eq!(block.kind, "shared build cache");
21647 assert!(matches!(
21648 block.cause,
21649 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
21650 ));
21651 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
21652 assert!(st.active.is_empty());
21653 }
21654
21655 #[tokio::test]
21656 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
21657 crate::run::pin_test_home();
21658 let tmp = tempfile::tempdir().unwrap();
21659 let mut st = state(1);
21660 let results = run(
21661 vec![job(
21662 sh("alpha", "echo ok"),
21663 tmp.path(),
21664 Duration::from_secs(1),
21665 )],
21666 Arc::new(Semaphore::new(0)),
21667 None,
21668 &mut st,
21669 )
21670 .await;
21671 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
21672 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
21673 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
21674 }
21675
21676 #[tokio::test]
21677 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
21678 crate::run::pin_test_home();
21679 let tmp = tempfile::tempdir().unwrap();
21680 let mut st = state(60);
21681 let results = run(
21682 vec![job(
21683 sh("alpha", "sleep 30"),
21684 tmp.path(),
21685 Duration::from_secs(1),
21686 )],
21687 Arc::new(Semaphore::new(1)),
21688 None,
21689 &mut st,
21690 )
21691 .await;
21692 let out = &results[0].2;
21693 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
21694 let class = FailClass::of(out).expect("a failure");
21695 assert_eq!(class, FailClass::Timeout);
21696 assert!(should_hand_over(None, &class), "existing handover policy");
21697 assert!(st.resource_block.is_none());
21698 }
21699}