1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's network, Git access and repository coordination before \
526 retrying"
527 )
528}
529
530enum FixStep {
532 Resource,
533 Recorded { report_lost: bool },
534}
535
536struct FixClaim {
553 path: PathBuf,
554}
555
556impl FixClaim {
557 fn acquire(dir: &Path) -> Result<Self> {
558 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559 let path = dir.join("fix.lock");
560 let retry = crate::lock_file::Retry::new(None);
561 match Self::create(&path, retry) {
562 Ok(claim) => Ok(claim),
563 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564 if Self::reclaim_if_dead(&path) {
565 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566 } else {
567 bail!(
568 "another repair or `magi fix` is already running for this run ({} exists)",
569 path.display()
570 )
571 }
572 }
573 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574 }
575 }
576
577 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578 let mut f = retry.create(path)?;
579 use std::io::Write as _;
580 writeln!(f, "{}", std::process::id())?;
582 Ok(Self {
583 path: path.to_owned(),
584 })
585 }
586
587 fn reclaim_if_dead(path: &Path) -> bool {
591 let dead = std::fs::read_to_string(path)
592 .ok()
593 .and_then(|body| body.trim().parse::<u32>().ok())
594 .is_some_and(|pid| !crate::proc::pid_alive(pid));
595 dead && std::fs::remove_file(path).is_ok()
596 }
597}
598
599impl Drop for FixClaim {
600 fn drop(&mut self) {
601 let _ = std::fs::remove_file(&self.path);
602 }
603}
604
605struct DriverExecution<'a> {
608 runner: &'a mut Runner,
609 driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613 fn drop(&mut self) {
614 self.driver.release_local(&mut self.runner.state);
615 }
616}
617
618impl Runner {
619 pub async fn start(
621 repo: &Path,
622 instruction: String,
623 config: Config,
624 origin: Origin,
625 ) -> Result<Self> {
626 Self::start_naming(repo, instruction, "", config, origin).await
627 }
628
629 pub async fn start_naming(
632 repo: &Path,
633 instruction: String,
634 _also_scan: &str,
635 config: Config,
636 origin: Origin,
637 ) -> Result<Self> {
638 let repo = git::toplevel(repo).await?;
639 let missing = agent::missing_programs(&config.agents);
640 if !missing.is_empty() {
641 bail!(
642 "these agent programs are not on PATH: {}. Fix the roster in \
643 magi.toml or install them.",
644 missing.join(", ")
645 );
646 }
647 let base_branch =
648 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649 .await?;
650 let fetched_base = resolve_base(
651 &repo,
652 &base_branch,
653 &config.merge.remote,
654 config.coordination.as_ref(),
655 )
656 .await?;
657 let base_commit = fetched_base.commit.clone();
658 let roles = config.resolve_roles()?;
659 let max_parallel = config.graph.max_parallel.max(1);
660 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661 state.fetched_base = Some(fetched_base);
665 state.origin = Some(origin);
666 state.reserve()?;
667 state.event("start", format!("run {} created", state.id));
668 state.save()?;
669 Ok(Self {
670 state,
671 roles,
672 sem: Arc::new(Semaphore::new(max_parallel)),
673 pause: Pause::new(),
674 interrupt: Pause::new(),
675 })
676 }
677
678 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692 Self::review_taking_over(repo, branch, config, None, origin).await
693 }
694
695 pub async fn review_taking_over(
702 repo: &Path,
703 branch: &str,
704 config: Config,
705 takeover: Option<crate::handover::Takeover>,
706 origin: Origin,
707 ) -> Result<Self> {
708 let repo = git::toplevel(repo).await?;
709 let missing = agent::missing_programs(&config.agents);
710 if !missing.is_empty() {
711 bail!(
712 "these agent programs are not on PATH: {}. Fix the roster in \
713 magi.toml or install them.",
714 missing.join(", ")
715 );
716 }
717 let base_branch =
718 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719 .await?;
720 if base_branch == branch {
721 bail!("`{branch}` is the base branch; there is nothing to review against");
722 }
723 let fetched_base = resolve_base(
724 &repo,
725 &base_branch,
726 &config.merge.remote,
727 config.coordination.as_ref(),
728 )
729 .await?;
730 let base_commit = fetched_base.commit.clone();
731
732 let roles = config.resolve_roles()?;
733 let max_parallel = config.graph.max_parallel.max(1);
734 let mut state = RunState::new(
735 repo.clone(),
736 base_branch,
737 base_commit.clone(),
738 String::new(),
739 config,
740 );
741 state.fetched_base = Some(fetched_base);
742 state.origin = Some(origin);
743 state.reserve()?;
744
745 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752 earlier: Vec::new(),
753 home: crate::run::home(),
754 choice: None,
755 task_instruction: None,
756 });
757 let wait = repository_wait(&state, None);
758 let coordination_config = state.config.coordination.clone();
759 let coordination =
760 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761 .await?;
762 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763 if let Some(released) = &released {
764 state.event(
765 "release",
766 format!(
767 "took `{branch}` over from run {}: its worktree was released: {}",
768 crate::run::short_of(&released.old_id),
769 released.audit
770 ),
771 );
772 }
773 if let Some(choice) = takeover.choice.as_ref()
777 && let Err(e) =
778 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779 .await
780 {
781 if let Some(released) = &released {
782 released.restore(&repo, branch).await;
783 }
784 return Err(e.context("applying the owner's answer about the diverged branch"));
785 }
786 for id in takeover.earlier.iter().rev() {
792 if let Ok(old) = RunState::load_under(id, &takeover.home)
793 && let Some(head) = old.pr_head.clone()
794 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795 {
796 state.pr_head = Some(head);
797 break;
798 }
799 }
800 drop(coordination);
801 let opened = Self::open_review(
802 &repo,
803 branch,
804 state,
805 roles,
806 max_parallel,
807 base_commit,
808 &takeover,
809 )
810 .await;
811 if opened.is_err()
812 && let Some(released) = &released
813 {
814 let _coordination = crate::repo_lock::SeatLease::operation_with(
815 &repo,
816 coordination_config.as_ref(),
817 &wait,
818 )
819 .await?;
820 released.restore(&repo, branch).await;
821 }
822 opened
823 }
824
825 pub async fn recover_pr(
827 repo: &Path,
828 task: &str,
829 selection: crate::pr_recovery::Selection,
830 mut config: Config,
831 ) -> Result<Self> {
832 selection.configure(&mut config)?;
833 selection.check_task(task, &selection.instruction)?;
834 let base_branch =
835 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836 .await?;
837 let fetched_base = resolve_base(
838 repo,
839 &base_branch,
840 &config.merge.remote,
841 config.coordination.as_ref(),
842 )
843 .await?;
844 let base_commit = fetched_base.commit.clone();
845 selection
846 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847 .await?;
848 Box::pin(selection.verify_source(repo)).await?;
849 if let Some(proof) = &selection.source_proof {
850 anyhow::ensure!(
851 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852 "source rebase base is outside the fetched landing base"
853 );
854 }
855 let roles = config.resolve_roles()?;
856 let max_parallel = config.graph.max_parallel.max(1);
857 let mut state = RunState::new(
858 repo.to_path_buf(),
859 base_branch,
860 base_commit.clone(),
861 selection.instruction.clone(),
862 config,
863 );
864 state.fetched_base = Some(fetched_base);
865 state.origin = Some(crate::run::Origin::queue(task));
866 state.pr_head = Some(selection.branch.clone());
867 state.pr_recovery = Some(selection.clone());
868 state.reserve()?;
869 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870 state.discard_reserved();
872 return Err(e);
873 }
874 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875 }
876
877 async fn open_selected(
878 repo: &Path,
879 state: RunState,
880 roles: ResolvedRoles,
881 max_parallel: usize,
882 base_commit: String,
883 selection: crate::pr_recovery::Selection,
884 ) -> Result<Self> {
885 let mut state = state;
886 let branch = state.branch_for('A');
887 let wait = repository_wait(&state, None);
888 let prepared = async {
889 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890 git::git(repo, &["branch", &branch, selection.source()]).await?;
891 drop(coordination);
892 anyhow::Ok(())
893 }
894 .await;
895 if let Err(e) = prepared {
896 state.discard_reserved();
899 return Err(e);
900 }
901 state.event(
902 "recovery",
903 format!(
904 "selected implementation {} for {} (public lease {})",
905 selection.source(),
906 selection.url,
907 selection.head
908 ),
909 );
910 let takeover = crate::handover::Takeover {
911 earlier: Vec::new(),
912 home: crate::run::home(),
913 choice: None,
914 task_instruction: Some(selection.instruction.clone()),
915 };
916 if let Err(e) = state.save() {
917 state.discard_reserved();
918 return Err(e);
919 }
920 let id = state.id.clone();
921 let opened = Self::open_review(
922 repo,
923 &branch,
924 state,
925 roles,
926 max_parallel,
927 base_commit,
928 &takeover,
929 )
930 .await;
931 match opened {
932 Ok(runner) => Ok(runner),
933 Err(e) => {
934 if let Ok(mut partial) = RunState::load(&id) {
935 partial.status = RunStatus::Blocked;
936 partial.driver_exited = true;
937 partial.event(
938 "recovery",
939 format!("startup failed; retained source and candidate {branch}: {e:#}"),
940 );
941 partial.save()?;
942 }
943 Err(crate::pr_recovery::StartupFailed {
944 run: id,
945 cause: format!("{e:#}"),
946 }
947 .into())
948 }
949 }
950 }
951
952 async fn inherit(
957 repo: &Path,
958 branch: &str,
959 base_commit: &str,
960 takeover: &crate::handover::Takeover,
961 ) -> Option<crate::recovery::RecoveryRecord> {
962 use crate::recovery::{Relation, assess};
963 let newest = takeover.earlier.last()?;
964 let mut chosen = None;
965 for id in takeover.earlier.iter().rev() {
966 if let Ok(old) = RunState::load_under(id, &takeover.home)
967 && !old.reviews.is_empty()
968 && (old.winner().is_some_and(|w| w.branch == branch)
969 || old.pr_head.as_deref() == Some(branch))
970 {
971 chosen = Some(old);
972 break;
973 }
974 }
975 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976 .await
977 .unwrap_or_default();
978 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979 Some(last) if last.head == tip => Relation::Same,
980 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981 _ => Relation::Diverged,
982 };
983 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984 Some(assess(from, chosen.as_ref(), base_commit, relation))
985 }
986
987 async fn open_review(
990 repo: &Path,
991 branch: &str,
992 mut state: RunState,
993 roles: ResolvedRoles,
994 max_parallel: usize,
995 base_commit: String,
996 takeover: &crate::handover::Takeover,
997 ) -> Result<Self> {
998 let wait = repository_wait(&state, None);
999 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000 if state.pr_recovery.is_none() {
1001 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002 }
1003 let start = review_base(
1006 repo,
1007 &state.config.merge.remote,
1008 &state.base_branch,
1009 &base_commit,
1010 branch,
1011 )
1012 .await;
1013 let log = git::log_oneline(repo, &start, branch)
1014 .await
1015 .unwrap_or_default();
1016 let log_text = if log.trim().is_empty() {
1017 "(no commit messages)"
1018 } else {
1019 log.trim()
1020 };
1021 let mut instruction = format!(
1022 "Review the work already on branch `{branch}`. There is no task \
1023 statement: what the change claims to do is whatever its commits \
1024 say.\n\n{log_text}"
1025 );
1026 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030 if let Some(task) = takeover
1031 .task_instruction
1032 .as_deref()
1033 .filter(|t| !t.trim().is_empty())
1034 && rec.reviewed_head.is_some()
1035 {
1036 instruction = format!(
1037 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038 task.trim()
1039 );
1040 rec.criteria_inherited = true;
1041 }
1042 state.event("recovery", rec.summary());
1043 state.recovery = Some(rec);
1044 }
1045 if let Some(selected) = &state.pr_recovery {
1046 instruction = selected.instruction.clone();
1047 }
1048 state.task_instruction = takeover.task_instruction.clone();
1049 state.instruction = instruction;
1050 state.reviewed_commits = Some(
1051 git::subjects(repo, &start, branch)
1052 .await
1053 .unwrap_or_default(),
1054 );
1055
1056 for id in &takeover.earlier {
1059 if let Ok(old) = RunState::load_under(id, &takeover.home)
1060 && old.repo == state.repo
1061 && (old.pr_head.as_deref() == Some(branch)
1062 || old.winner().is_some_and(|w| w.branch == branch))
1063 {
1064 for hold in old.branch_owner_holds {
1065 if !state.branch_owner_holds.contains(&hold) {
1066 state.event(
1067 "approval-policy",
1068 format!(
1069 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070 hold.control.key(),
1071 hold.file,
1072 hold.head
1073 ),
1074 );
1075 state.branch_owner_holds.push(hold);
1076 }
1077 }
1078 }
1079 }
1080 let reviewed_head = git::rev_parse(repo, branch).await?;
1081 state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083 let worktree = state.worktree_root().join("under-review");
1086 if let Some(parent) = worktree.parent() {
1087 tokio::fs::create_dir_all(parent).await.ok();
1088 }
1089 let path = worktree.to_string_lossy().to_string();
1090 git::git(repo, &["worktree", "add", &path, branch])
1091 .await
1092 .with_context(|| {
1093 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094 })?;
1095
1096 let setup = match ensure_setup_config(&mut state, repo).await {
1097 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098 Err(e) => Err(e),
1099 };
1100 if let Err(e) = setup {
1101 git::worktree_remove_clean(repo, &worktree).await.ok();
1102 return Err(e);
1103 }
1104 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105 .await
1106 .unwrap_or(0);
1107 if commits == 0 {
1108 git::worktree_remove_clean(repo, &worktree).await.ok();
1109 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110 }
1111 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112 .await
1113 .map(|f| f.len())
1114 .unwrap_or(0);
1115 if files == 0
1116 && let (Ok(head_tree), Ok(base_tree)) = (
1117 git::tree_of(&worktree, "HEAD").await,
1118 git::tree_of(&worktree, &base_commit).await,
1119 )
1120 && head_tree == base_tree
1121 {
1122 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123 git::worktree_remove_clean(repo, &worktree).await.ok();
1124 bail!(
1125 "`{branch}` at {} has a tree identical to base {}; this usually means \
1126 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127 against `{}/{branch}`) rather than an empty change",
1128 short(&head),
1129 short(&base_commit),
1130 state.config.merge.remote
1131 );
1132 }
1133 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134 .await
1135 .unwrap_or_default();
1136
1137 state.candidates.push(Candidate {
1138 index: 0,
1139 label: 'A',
1140 agent: EXISTING_BRANCH.to_owned(),
1143 branch: branch.to_owned(),
1144 worktree,
1145 summary: String::new(),
1146 stat,
1147 files,
1148 commits,
1149 empty: false,
1150 failed: None,
1151 verified_noop: None,
1152 duration_ms: 0,
1153 folded: false,
1154 });
1155 state.tally = Some(Tally {
1156 first_choice: BTreeMap::from([('A', 0)]),
1157 borda: BTreeMap::new(),
1158 winner: 'A',
1159 rankings: 0,
1160 unanimous_initial: false,
1161 deliberated: false,
1162 changed_votes: 0,
1163 unanimous_final: false,
1164 tie_break: None,
1165 judges: 0,
1169 present: 0,
1170 quorum: 0,
1171 met_quorum: true,
1172 uncontested: Some("review-only run: nothing competed".to_owned()),
1173 });
1174 state.status = RunStatus::Reviewing;
1175 state.prep_complete = Some(true);
1176 state.event(
1177 "start",
1178 format!(
1179 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180 state.id
1181 ),
1182 );
1183 state.save()?;
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1196 Self::authorize_base_recovery_request(home, id, task, false).await
1197 }
1198
1199 pub(crate) async fn authorize_base_recovery_from_resumption(
1201 home: &Path,
1202 id: &str,
1203 task: &str,
1204 ) -> Result<()> {
1205 Self::authorize_base_recovery_request(home, id, task, true).await
1206 }
1207
1208 async fn authorize_base_recovery_request(
1209 home: &Path,
1210 id: &str,
1211 task: &str,
1212 resumption: bool,
1213 ) -> Result<()> {
1214 let _owner = crate::pr_head::owner_lock(home, id)?;
1215 let mut state = RunState::load_under(id, home)?;
1216 crate::pr_head::ensure_owner(&state)?;
1217 let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1218 anyhow::ensure!(
1219 queued.runs.last().is_some_and(|run| run == id)
1220 && queued.status != crate::queue::TaskStatus::Done
1221 && !queued.fresh_start
1222 && queued.review_branch.is_none(),
1223 "run task ownership changed"
1224 );
1225 anyhow::ensure!(
1226 state
1227 .origin
1228 .as_ref()
1229 .and_then(|o| o.task.as_deref())
1230 .is_none_or(|owner| owner == task),
1231 "run belongs to another task"
1232 );
1233 let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1234 anyhow::ensure!(
1235 state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1236 "run is live or its ownership is unknown"
1237 );
1238 let _execution = crate::driver::maintenance(home, id)?;
1241 anyhow::ensure!(
1242 state.status.resumable()
1243 && state.released_to.is_none()
1244 && state.retired_to.is_none()
1245 && state.pr_taken_over_by.is_none(),
1246 "run ownership is final"
1247 );
1248 anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1249 anyhow::ensure!(
1250 !state.pr_publication_pending
1251 && !crate::handover::landing_checkpoint(&state)
1252 && state.merge_decision.is_none(),
1253 "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1254 );
1255 let winner = state.winner().context("run has no retained winner")?;
1256 anyhow::ensure!(
1257 git::is_clean(&winner.worktree).await?,
1258 "retained worktree has uncommitted changes"
1259 );
1260 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1261 anyhow::ensure!(
1262 git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1263 "retained worktree does not match the winner branch"
1264 );
1265 state.record_base_sync_stop(&head);
1266 state.save()?;
1267 if resumption {
1268 anyhow::ensure!(
1269 crate::daemon::compatible_run_instruction(&state, &queued),
1270 "resumption instruction does not match the retained run"
1271 );
1272 let approval = queued
1273 .resumption
1274 .as_ref()
1275 .context("no operator resumption")?;
1276 anyhow::ensure!(
1277 approval.instruction == queued.instruction,
1278 "resumption approval does not match the current task instruction"
1279 );
1280 state.authorize_base_sync_resumption(task, approval)?;
1281 } else {
1282 state.authorize_base_sync(task)?;
1283 }
1284 state.event(
1285 "base-recovery",
1286 "owner authorized one additional base rebase; fresh verification required",
1287 );
1288 state.save()?;
1289 Ok(())
1290 }
1291
1292 pub fn resume(id: &str) -> Result<Self> {
1294 let state = RunState::load(id)?;
1295 crate::pr_head::ensure_owner(&state)?;
1296 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1297 bail!(
1298 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1299 );
1300 }
1301 if let Some(to) = &state.released_to {
1302 bail!(
1303 "run {} cannot be resumed: its worktree was released to run {}",
1304 state.short(),
1305 crate::run::short_of(to)
1306 );
1307 }
1308 let roles = state.config.resolve_roles()?;
1309 if let Some(id) = &state.config.graph.selected_implementer
1310 && state.candidates.iter().any(|c| c.agent != *id)
1311 {
1312 bail!(
1313 "replacement implementer pin does not match the saved candidates; task must stay held"
1314 );
1315 }
1316 let max_parallel = state.config.graph.max_parallel.max(1);
1317 Ok(Self {
1318 state,
1319 roles,
1320 sem: Arc::new(Semaphore::new(max_parallel)),
1321 pause: Pause::new(),
1322 interrupt: Pause::new(),
1323 })
1324 }
1325
1326 pub async fn execute(&mut self) -> Result<()> {
1333 if let Some(selection) = &self.state.pr_recovery
1334 && (selection.source_proof.is_some() || selection.source_run.is_some())
1335 {
1336 Box::pin(selection.verify_source(&self.state.repo)).await?;
1337 }
1338 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1341 || self.state.land_approval.is_some()
1342 || self.state.merge_decision.is_some()
1343 || !self.state.merge_repairs.is_empty()
1344 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1345 {
1346 let claim = FixClaim::acquire(&self.state.dir())?;
1347 let disk = RunState::load(&self.state.id)?;
1348 let checkpoint = |s: &RunState| {
1349 (
1350 s.merge_repairs.len(),
1351 s.merge_repairs
1352 .last()
1353 .map(|r| (r.question.clone(), r.phase)),
1354 s.land_approval
1355 .as_ref()
1356 .map(|a| (a.question.clone(), a.head.clone())),
1357 )
1358 };
1359 if checkpoint(&disk) != checkpoint(&self.state) {
1360 bail!(
1361 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1362 );
1363 }
1364 Some(claim)
1365 } else {
1366 None
1367 };
1368 let mut execution = self.claim_driver().await?;
1369 let runner = &mut *execution.runner;
1370 let wait = runner.repository_wait();
1371 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1374 let result = match result {
1375 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1376 record_repository_block(&mut runner.state, &e);
1377 runner.park_here()?;
1378 Ok(())
1379 }
1380 other => other,
1381 };
1382 execution.driver.finish(&mut runner.state).await?;
1383 let ended = if result.is_err() {
1384 Some({
1385 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1386 if let Err(error) = &result {
1387 notice.message = format!(
1388 "Run {} stopped with an error: {}",
1389 runner.state.id,
1390 error.to_string().lines().next().unwrap_or("unknown error")
1391 );
1392 }
1393 notice
1394 })
1395 } else {
1396 crate::notices::run_ended(&runner.state)
1397 };
1398 if let Some(notice) = ended {
1399 crate::notices::raise(notice);
1400 }
1401 result
1402 }
1403
1404 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1405 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1406 }
1407
1408 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1409 let wait = self.repository_wait();
1410 coordinate_operation(&mut self.state, cwd, &wait).await
1411 }
1412
1413 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1415 let owner = match crate::pr_head::owner_lock_wait(
1416 &crate::run::home(),
1417 &self.state.id,
1418 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1419 )
1420 .await
1421 {
1422 Ok(owner) => owner,
1423 Err(error) => {
1424 let detail = format!("worktree maintenance or ownership update: {error:#}");
1425 self.state.resource_block = Some(crate::run::ResourceBlock {
1429 cause: crate::run::ResourceBlockCause::Other,
1430 node: "start".into(),
1431 kind: "worktree maintenance".into(),
1432 detail: detail.clone(),
1433 at: Timestamp::now(),
1434 });
1435 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1436 }
1437 };
1438 let disk = RunState::load(&self.state.id)?;
1439 crate::pr_head::ensure_owner(&disk)?;
1440 let claimed =
1441 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1442 anyhow::ensure!(
1447 disk.liveness(claimed) != crate::run::Liveness::Live,
1448 "run already has a live driver; wait for its execution to end"
1449 );
1450 if let Some(to) = &disk.released_to {
1451 bail!(
1452 "run {} cannot continue: its worktree was released to run {}",
1453 self.state.short(),
1454 crate::run::short_of(to)
1455 );
1456 }
1457 anyhow::ensure!(
1458 serde_json::to_value(&disk.base_sync_recovery)?
1459 == serde_json::to_value(&self.state.base_sync_recovery)?,
1460 "base recovery changed since this run was loaded; reload the saved run"
1461 );
1462 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1463 let mut execution = DriverExecution {
1464 runner: self,
1465 driver,
1466 };
1467 let _owner = owner;
1470 let state = &mut execution.runner.state;
1471 state.parked = false;
1476 state.resource_wait = None;
1477 state.resource_block = None;
1478 state.clear_active();
1485 let pid = std::process::id();
1500 state.driver_pid = Some(pid);
1501 state.driver_started_at = crate::proc::process_started_at(pid);
1502 state.driver_exited = false;
1503 state.driver_claimed = Some(true);
1504 state.begin_pass();
1505 execution.driver.bind(state);
1506 state.save()?;
1507 execution.driver.record(state)?;
1508 Ok(execution)
1509 }
1510
1511 async fn execute_graph(&mut self) -> Result<()> {
1512 self.state.configure_recovery_resume()?;
1513 self.persist_candidates().await?;
1516 if self.state.base_sync_blocked() {
1519 self.sync_to_base().await?;
1520 if !self.state.base_sync_blocked()
1521 && self.state.status != RunStatus::AlreadyInBase
1522 && !self.resource_halt()?
1523 {
1524 self.finish_after_tally().await?;
1525 }
1526 return Ok(());
1527 }
1528 if self
1529 .state
1530 .merge_decision
1531 .as_ref()
1532 .is_some_and(|decision| !decision.closed)
1533 && self.state.status != RunStatus::Landing
1534 {
1535 if crate::merge_decision::resumable_hold(&self.state) {
1536 self.state.merge = Some(MergeOutcome {
1537 mode: MergeMode::Pr,
1538 ok: false,
1539 detail: self
1540 .state
1541 .merge_decision
1542 .as_ref()
1543 .unwrap()
1544 .target
1545 .pr
1546 .clone(),
1547 empty: false,
1548 });
1549 self.run_land().await?;
1550 self.settle_questions();
1551 return Ok(());
1552 }
1553 bail!(
1554 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1555 );
1556 }
1557 if self.state.config.graph.selected_implementer.is_some()
1558 && self.state.status == RunStatus::Blocked
1559 && self.state.tally.is_none()
1560 && self.state.candidate_preservation.is_empty()
1561 && self.state.interrupted_implementations.is_empty()
1562 {
1563 bail!(
1564 "selected implementation is held; explicitly release the owning task to retry it"
1565 );
1566 }
1567
1568 if self.state.merge_repairs.last().is_some_and(|r| {
1569 matches!(
1570 r.phase,
1571 crate::run::MergeRepairPhase::Requested
1572 | crate::run::MergeRepairPhase::Fixing
1573 | crate::run::MergeRepairPhase::Reviewing
1574 )
1575 }) {
1576 if self.repair_merge_review().await? {
1577 self.run_land().await?;
1578 }
1579 self.settle_questions();
1580 return Ok(());
1581 }
1582 if self.state.pr_publication_pending {
1585 self.merge().await?;
1586 if self.resource_halt()? {
1587 return Ok(());
1588 }
1589 self.state.save()?;
1590 self.settle_questions();
1591 return Ok(());
1592 }
1593 if self.state.status == RunStatus::Stalled {
1606 if self.recover_stall().await? {
1607 self.finish_after_tally().await?;
1608 } else {
1609 self.state.save()?;
1611 }
1612 return Ok(());
1613 }
1614 if self.state.status == RunStatus::Landing
1624 || crate::handover::landing_checkpoint(&self.state)
1625 {
1626 self.state.land_sync_pending = true;
1627 self.state.save()?;
1628 self.run_land().await?;
1629 self.settle_questions();
1634 return Ok(());
1635 }
1636 if self.state.github_text.is_some()
1641 && self.state.merge.is_none()
1642 && self.state.status == RunStatus::Gating
1643 {
1644 self.merge().await?;
1645 if self.resource_halt()? {
1646 return Ok(());
1647 }
1648 self.state.save()?;
1649 return Ok(());
1650 }
1651 if let Err(e) = self.prep().await {
1652 if e.is::<crate::repo_lock::ResourceBlocked>() {
1653 return Err(e);
1654 }
1655 self.state.status = RunStatus::Blocked;
1656 self.state
1657 .prep_failure
1658 .get_or_insert_with(|| format!("{e:#}"));
1659 self.state
1660 .event("prep", format!("preparation stopped: {e:#}"));
1661 self.state.save()?;
1662 return Err(e);
1663 }
1664 if self.park_here()? {
1665 return Ok(());
1666 }
1667 self.advise().await?;
1668 if self.park_here()? {
1669 return Ok(());
1670 }
1671 self.implement().await?;
1672 if self.park_here()? {
1673 return Ok(());
1674 }
1675 if self.state.status == RunStatus::VerifiedNoop {
1679 return Ok(());
1680 }
1681 self.judge().await?;
1682 if self.park_here()? {
1683 return Ok(());
1684 }
1685 self.deliberate().await?;
1686 if self.park_here()? {
1687 return Ok(());
1688 }
1689 self.vote().await?;
1690 if self.park_here()? {
1691 return Ok(());
1692 }
1693 self.tally()?;
1694 if self.state.status == RunStatus::Stalled {
1699 self.state.save()?;
1703 return Ok(());
1704 }
1705 self.finish_after_tally().await?;
1706 Ok(())
1707 }
1708
1709 fn stop_for_resource(&mut self) -> Result<()> {
1714 if self.pause.parked() || self.interrupt.parked() {
1717 self.park_here()?;
1718 return Ok(());
1719 }
1720 if self.state.resource_block.is_some() {
1721 self.state.status = RunStatus::Blocked;
1722 }
1723 self.state.save()
1724 }
1725
1726 fn resource_halt(&mut self) -> Result<bool> {
1728 if self.state.resource_block.is_none() {
1729 return Ok(false);
1730 }
1731 self.stop_for_resource()?;
1732 Ok(true)
1733 }
1734
1735 fn park_here(&mut self) -> Result<bool> {
1742 let parking = self.pause.parked() || self.interrupt.parked();
1743 if !parking && self.state.resource_block.is_some() {
1744 self.stop_for_resource()?;
1745 return Ok(true);
1746 }
1747 if !self.pause.parked() && !self.interrupt.parked() {
1752 return Ok(false);
1753 }
1754 if self.state.parked {
1755 return Ok(true);
1756 }
1757 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1758 if let Some(block) = &mut self.state.resource_block
1759 && matches!(
1760 block.cause,
1761 crate::run::ResourceBlockCause::Cancelled { .. }
1762 )
1763 {
1764 block.cause = crate::run::ResourceBlockCause::Cancelled {
1765 park: cause.clone(),
1766 };
1767 }
1768 let why = match cause.as_ref().map(ParkCause::label) {
1769 Some(reason) => format!(
1770 "parked after `{}` ({reason}) — resume to carry on from here",
1771 self.state.status.as_str()
1772 ),
1773 None => format!(
1774 "parked after `{}` — resume to carry on from here",
1775 self.state.status.as_str()
1776 ),
1777 };
1778 self.state.event("park", why);
1779 self.state
1780 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1781 self.state.parked = true;
1782 self.state.save()?;
1783 Ok(true)
1784 }
1785
1786 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1788 if self.state.prep_failure.is_none() {
1789 return Ok(());
1790 }
1791 let failed_at = self
1792 .state
1793 .events
1794 .iter()
1795 .rev()
1796 .find(|event| event.node == "prep")
1797 .map_or(self.state.updated_at, |event| event.at);
1798 anyhow::ensure!(
1799 approval_at > failed_at,
1800 "preparation retry requires a new explicit release after the failure"
1801 );
1802 self.state.prep_failure = None;
1803 self.state.prep_complete = Some(false);
1804 self.state
1805 .event("prep-retry", "operator approved retry of preparation");
1806 Ok(())
1807 }
1808
1809 pub fn on_pause(&mut self, pause: Pause) {
1811 self.pause = pause;
1812 }
1813
1814 pub fn watch_interrupt(&mut self, pause: Pause) {
1820 self.interrupt = pause;
1821 }
1822
1823 fn settle_questions(&mut self) {
1843 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1844 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1845 }
1846 }
1847
1848 async fn finish_after_tally(&mut self) -> Result<()> {
1851 self.graft().await?;
1854 if self.resource_halt()? {
1855 return Ok(());
1856 }
1857 self.fold_losers().await?;
1858 self.sync_to_base().await?;
1863 if self.hold_base_sync().await? || self.resource_halt()? {
1864 return Ok(());
1865 }
1866 if self.state.status == RunStatus::AlreadyInBase {
1867 return Ok(());
1868 }
1869 self.review_loop().await?;
1870 if self.resource_halt()? {
1871 return Ok(());
1872 }
1873 self.sync_to_base().await?;
1874 if self.hold_base_sync().await? || self.resource_halt()? {
1875 return Ok(());
1876 }
1877 if self.state.status == RunStatus::AlreadyInBase {
1878 return Ok(());
1879 }
1880 self.gate().await?;
1881 self.merge().await?;
1882 if self.resource_halt()? {
1883 return Ok(());
1884 }
1885 self.state.save()?;
1886 Ok(())
1887 }
1888
1889 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1896 let result = match ensure_setup_config(&mut self.state, repo).await {
1897 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1898 Err(e) => Err(e),
1899 };
1900 if let Err(e) = result {
1901 git::worktree_remove_clean(repo, wt).await.ok();
1902 if in_prep {
1903 self.rollback_prep(repo).await;
1904 }
1905 self.state.status = RunStatus::Blocked;
1906 self.state.event(
1907 "setup",
1908 format!("worktree setup failed in {}: {e:#}", wt.display()),
1909 );
1910 self.state.save()?;
1911 return Err(e);
1912 }
1913 Ok(())
1914 }
1915
1916 async fn rollback_prep(&mut self, repo: &Path) {
1918 let root = self.state.worktree_root();
1919 for c in std::mem::take(&mut self.state.candidates) {
1920 if matches!(
1921 git::worktree_remove_clean(repo, &c.worktree).await,
1922 Ok(true)
1923 ) {
1924 git::branch_delete(repo, &c.branch).await.ok();
1925 } else {
1926 self.state.candidates.push(c);
1927 }
1928 }
1929 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1930 for j in 1..=seats {
1931 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1932 .await
1933 .ok();
1934 }
1935 for k in 1..=self.state.config.graph.advisors {
1936 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1937 .await
1938 .ok();
1939 }
1940 }
1941
1942 async fn prep(&mut self) -> Result<()> {
1943 if let Some(reason) = &self.state.prep_failure {
1944 bail!(
1945 "candidate preparation previously failed: {reason}; correct the source before retrying"
1946 );
1947 }
1948 if self
1949 .state
1950 .prep_complete
1951 .unwrap_or(!self.state.candidates.is_empty())
1952 {
1953 return Ok(());
1954 }
1955 let repo = self.state.repo.clone();
1956 let _coordination = self.coordinate(&repo).await?;
1959 let base = self.state.base_commit.clone();
1960 let plan = self.state.sources.plan(&repo, &base).await?;
1961 self.rollback_prep(&repo).await;
1962 if !self.state.candidates.is_empty() {
1963 bail!(
1964 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1965 );
1966 }
1967 self.state.prep_complete = Some(false);
1968 self.state.status = RunStatus::Prep;
1969 self.state.save()?;
1970 let repo = self.state.repo.clone();
1971 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1972 let root = self.state.worktree_root();
1973 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1974
1975 let hooks_dir = self.state.dir().join("hooks");
1978 if self.state.config.blind.commit_msg_hook {
1979 std::fs::create_dir_all(&hooks_dir)
1980 .with_context(|| format!("create {}", hooks_dir.display()))?;
1981 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1982 let path = hooks_dir.join("commit-msg");
1983 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1984 make_executable(&path)?;
1985 git::acquire_worktree_config(&repo).await?;
1993 self.state.enabled_worktree_config = true;
1994 }
1995
1996 ensure_setup_config(&mut self.state, &repo).await?;
1999
2000 for (index, (spec, label)) in self
2001 .roles
2002 .implementers
2003 .clone()
2004 .into_iter()
2005 .zip(labels)
2006 .enumerate()
2007 {
2008 let branch = self.state.branch_for(label);
2009 let worktree = root.join(format!("cand-{label}"));
2010 self.state.event(
2014 "prep-intent",
2015 serde_json::json!({
2016 "branch": branch,
2017 "worktree": worktree,
2018 "start": start,
2019 "picks": plan.picks,
2020 })
2021 .to_string(),
2022 );
2023 self.state.save()?;
2024 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2025 if self.state.config.blind.commit_msg_hook {
2026 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2027 }
2028 git::local_exclude(&worktree, "/.magi/").await?;
2029 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2030 git::worktree_remove_clean(&repo, &worktree).await.ok();
2033 git::branch_delete(&repo, &branch).await.ok();
2034 self.rollback_prep(&repo).await;
2035 self.state.status = RunStatus::Blocked;
2036 self.state
2037 .event("prep", format!("worktree setup failed: {e:#}"));
2038 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2039 self.state.save()?;
2040 return Err(e);
2041 }
2042 self.state.candidates.push(Candidate {
2043 index,
2044 label,
2045 agent: spec.id.clone(),
2046 branch,
2047 worktree,
2048 summary: String::new(),
2049 stat: String::new(),
2050 files: 0,
2051 commits: 0,
2052 empty: false,
2053 failed: None,
2054 verified_noop: None,
2055 duration_ms: 0,
2056 folded: false,
2057 });
2058 self.state.save()?;
2059 for pick in &plan.picks {
2060 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2061 {
2062 self.state.status = RunStatus::Blocked;
2063 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2064 self.state
2065 .event("prep", format!("cannot apply referenced commit: {e}"));
2066 self.state.save()?;
2067 return Err(e);
2068 }
2069 }
2070 }
2071
2072 for j in 1..=self.roles.judges.len() {
2073 let wt = root.join(format!("judge-{j}"));
2074 if !wt.exists() {
2075 git::worktree_add_detached(&repo, &wt, &base).await?;
2076 self.setup_seat_worktree(&repo, &wt, true).await?;
2077 }
2078 }
2079
2080 if self.state.config.graph.advise {
2088 for k in 1..=self.state.config.graph.advisors {
2089 let wt = root.join(format!("advisor-{k}"));
2090 if !wt.exists() {
2091 git::worktree_add_detached(&repo, &wt, &base).await?;
2092 self.setup_seat_worktree(&repo, &wt, true).await?;
2093 }
2094 }
2095 }
2096
2097 let authors: Vec<&str> = self
2102 .roles
2103 .implementers
2104 .iter()
2105 .map(|a| a.id.as_str())
2106 .collect();
2107 let overlap: Vec<String> = self
2108 .roles
2109 .judges
2110 .iter()
2111 .enumerate()
2112 .filter(|(_, j)| authors.contains(&j.id.as_str()))
2113 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2114 .collect();
2115 if !overlap.is_empty() {
2116 let note = format!(
2117 "{} also authored a candidate; blind, but the panel is less \
2118 independent than {} distinct agents would be",
2119 overlap.join(", "),
2120 self.roles.judges.len()
2121 );
2122 self.state.event("prep", note);
2123 }
2124
2125 self.state.event(
2126 "prep",
2127 format!(
2128 "{} candidates, {} judges, base {} ({})",
2129 self.state.candidates.len(),
2130 self.roles.judges.len(),
2131 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2132 self.state.base_branch
2133 ),
2134 );
2135 self.state.prep_complete = Some(true);
2136 self.state.status = RunStatus::Implementing;
2137 self.state.save()?;
2138 Ok(())
2139 }
2140
2141 async fn advise(&mut self) -> Result<()> {
2174 let implement_untouched = self
2175 .state
2176 .candidates
2177 .iter()
2178 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2179 if !self.state.config.graph.advise || self.state.advise_attempted {
2180 return Ok(());
2181 }
2182 if !implement_untouched {
2183 self.state.event(
2184 "advise",
2185 "skipping the design-deliberation stage: at least one \
2186 candidate already shows implementation progress, so this \
2187 run is past the point the stage exists to run before"
2188 .to_owned(),
2189 );
2190 self.state.advise_attempted = true;
2191 self.state.save()?;
2192 return Ok(());
2193 }
2194 let run_id = self.state.id.clone();
2195 let prompts = self.state.config.prompts.clone();
2196 let instruction = self.state.instruction.clone();
2197 let language = self.state.config.graph.language.clone();
2198 let root = self.state.worktree_root();
2199 let n = self.state.config.graph.advisors;
2200 let where_recorded = self.state.dir().join("run.json");
2201
2202 let seats = match self.state.config.advisors() {
2203 Ok(seats) if !seats.is_empty() => seats,
2204 Ok(_) => {
2205 self.state.event(
2206 "advise",
2207 format!(
2208 "[graph] advisors is 0; skipping the design-deliberation \
2209 stage and continuing without a synthesis brief (see {})",
2210 where_recorded.display()
2211 ),
2212 );
2213 self.state.advise_attempted = true;
2214 self.state.save()?;
2215 return Ok(());
2216 }
2217 Err(e) => {
2218 self.state.event(
2219 "advise",
2220 format!(
2221 "could not resolve advisor seats ({e:#}); continuing \
2222 without a design-deliberation brief (see {})",
2223 where_recorded.display()
2224 ),
2225 );
2226 self.state.advise_attempted = true;
2227 self.state.save()?;
2228 return Ok(());
2229 }
2230 };
2231
2232 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2233 let artifacts = agent::artifacts_dir(&self.state.dir());
2234 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2235
2236 let mut jobs = Vec::new();
2237 for (i, spec) in seats.iter().cloned().enumerate() {
2238 let seat_key = format!("advisor-{}", i + 1);
2239 let seat = self.seat(&seat_key, &spec.id);
2240 jobs.push(SeatJob {
2241 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2242 spec,
2243 seat,
2244 cwd: worktrees[i % worktrees.len()].clone(),
2245 timeout,
2246 allow_write: false,
2247 sessions: false,
2248 artifacts: artifacts.clone(),
2249 stem: seat_key,
2250 handover: None,
2251 });
2252 }
2253
2254 self.state.event(
2255 "advise",
2256 format!(
2257 "{} advisor seat(s) sketching a design in parallel",
2258 jobs.len()
2259 ),
2260 );
2261 let mut quota_losses = Vec::new();
2262 let cache = self.state.config.cache_dir();
2263 let ctx = WaveCtx {
2264 carry_seats: false,
2265 pauses: Some((&self.pause, &self.interrupt)),
2266 run: &run_id,
2267 node: "advise",
2268 prompts: &prompts,
2269 cache: cache.as_deref(),
2270 round: None,
2271 };
2272 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2273 let results = ask_json_wave::<Proposal>(
2274 jobs,
2275 Arc::clone(&self.sem),
2276 self.state.config.graph.retries,
2277 &advisor_roster,
2278 &ctx,
2279 &mut quota_losses,
2280 &mut self.state,
2281 &|p: &Proposal| p.validate(),
2282 )
2283 .await;
2284 if self.state.resource_block.is_some() {
2285 return self.stop_for_resource();
2286 }
2287 self.state.quota.extend(quota_losses);
2288
2289 let mut records = Vec::with_capacity(results.len());
2290 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2291 let agent_id = seat.agent.clone();
2292 self.state.seats.insert(seat.key.clone(), seat);
2293 match res {
2294 Ok((proposal, out)) => {
2295 self.state
2296 .event("advise", format!("advisor-{} proposed a design", i + 1));
2297 records.push(advise::AdvisorRecord::proposed(
2298 i + 1,
2299 agent_id,
2300 proposal,
2301 out.duration_ms,
2302 ));
2303 }
2304 Err(e) => {
2305 self.state.event(
2306 "advise",
2307 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2308 );
2309 records.push(advise::AdvisorRecord::failed(
2310 i + 1,
2311 agent_id,
2312 e.to_string(),
2313 ));
2314 }
2315 }
2316 }
2317
2318 let mut advice = advise::Advice {
2319 records,
2320 synthesis: None,
2321 };
2322 if advice.proposals().is_empty() {
2323 self.state.event(
2324 "advise",
2325 "no advisor produced a usable proposal; continuing without a \
2326 synthesis brief"
2327 .to_owned(),
2328 );
2329 } else {
2330 match self
2331 .synthesize_brief(
2332 &advice,
2333 &instruction,
2334 &language,
2335 &worktrees[0],
2336 &artifacts,
2337 &run_id,
2338 &prompts,
2339 cache.as_deref(),
2340 )
2341 .await
2342 {
2343 Ok(Some(text)) => {
2344 self.state.event(
2345 "advise",
2346 "synthesized a design brief for the implementer".to_owned(),
2347 );
2348 advice.synthesis = Some(text);
2349 }
2350 Ok(None) => {
2351 self.state.event(
2352 "advise",
2353 "the synthesis seat produced nothing usable; continuing \
2354 without a design brief"
2355 .to_owned(),
2356 );
2357 }
2358 Err(e) => {
2359 if e.is::<crate::repo_lock::ResourceBlocked>() {
2360 return Err(e);
2361 }
2362 self.state.event(
2363 "advise",
2364 format!("could not synthesize a design brief: {e:#}"),
2365 );
2366 }
2367 }
2368 }
2369 advise::apply_reflection(&mut advice);
2370
2371 self.state.advice = Some(advice);
2372 self.state.advise_attempted = true;
2373 self.state.save()?;
2374 Ok(())
2375 }
2376
2377 #[allow(clippy::too_many_arguments)]
2389 async fn synthesize_brief(
2390 &mut self,
2391 advice: &advise::Advice,
2392 instruction: &str,
2393 language: &str,
2394 cwd: &Path,
2395 artifacts: &Path,
2396 run_id: &str,
2397 prompts: &Prompts,
2398 cache: Option<&Path>,
2399 ) -> Result<Option<String>> {
2400 let chain = agent::pick_chain(
2401 &self.state.config.agents,
2402 self.state.config.roles.synthesizer.as_ref(),
2403 &agent::installed,
2404 "synthesizer",
2405 )?;
2406 let proposals = advice.proposals();
2407 let mut prompt = prompt::with_overlay(
2408 prompt::synthesize_brief(instruction, &proposals, language),
2409 prompts.overlay("advise"),
2410 );
2411 if cache.is_some() {
2412 prompt.push('\n');
2417 prompt.push_str(&prompt::build_cache_note("advise", false));
2418 }
2419 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2420 let mut last = None;
2425 for (n, spec) in chain.iter().enumerate() {
2426 if n > 0 {
2427 self.state
2428 .event("advise", format!("synthesis falling back to {}", spec.id));
2429 }
2430 let mut seat = self.seat("advise-synthesis", &spec.id);
2431 let wait = self.repository_wait();
2432 let outcome = wait
2433 .scope(agent::invoke(
2434 spec,
2435 &mut seat,
2436 &Invocation {
2437 coordination: self.state.config.coordination.as_ref(),
2438 fetched_base: None,
2439 cwd,
2440 prompt: &prompt,
2441 timeout,
2442 allow_write: false,
2443 sessions: false,
2444 artifacts,
2445 stem: &if n == 0 {
2446 "advise-synthesis".to_owned()
2447 } else {
2448 format!("advise-synthesis-{}", spec.id)
2449 },
2450 run: run_id,
2451 node: "advise",
2452 cache_dir: None,
2453 attachments: &[],
2454 writable: &[],
2455 },
2456 ))
2457 .await;
2458 if outcome.is_ok() {
2459 self.state.seats.insert(seat.key.clone(), seat);
2460 }
2461 let advance = agent::chain_advances(&outcome);
2462 last = Some(outcome);
2463 if !advance {
2464 break;
2465 }
2466 }
2467 let out = last.expect("a chain holds at least one agent")?;
2470 if !out.usable() {
2471 return Ok(None);
2472 }
2473 let text =
2474 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2475 Ok((!text.trim().is_empty()).then_some(text))
2476 }
2477
2478 async fn implement(&mut self) -> Result<()> {
2481 self.persist_candidates().await?;
2482 let run_id = self.state.id.clone();
2487 let prompts = self.state.config.prompts.clone();
2488 let todo: Vec<usize> = self
2489 .state
2490 .candidates
2491 .iter()
2492 .enumerate()
2493 .filter(|(_, c)| {
2494 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2495 && c.failed.is_none()
2496 && !c.empty
2497 })
2498 .map(|(i, _)| i)
2499 .collect();
2500 if todo.is_empty() {
2501 return self.after_implement();
2502 }
2503 self.state.status = RunStatus::Implementing;
2504
2505 let language = self.state.config.graph.language.clone();
2506 let github_language = self.state.config.graph.github_language.clone();
2507 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2508 let sessions = self.state.config.graph.sessions;
2509 let artifacts = agent::artifacts_dir(&self.state.dir());
2510 let brief = self
2514 .state
2515 .advice
2516 .as_ref()
2517 .and_then(|a| a.synthesis.as_deref())
2518 .map(str::to_owned);
2519 let attachments = self.state.attachments.clone();
2520
2521 let mut jobs = Vec::new();
2522 for &i in &todo {
2523 let (index, label, worktree) = {
2524 let c = &self.state.candidates[i];
2525 (c.index, c.label, c.worktree.clone())
2526 };
2527 let spec = self.roles.implementers[index].clone();
2528 let seat_key = format!("impl-{label}");
2529 let seat = self.seat(&seat_key, &spec.id);
2530 let instruction = seeded_instruction(&self.state);
2531 jobs.push(SeatJob {
2532 spec,
2533 seat,
2534 prompt: prompt::implement_in(
2535 &instruction,
2536 &worktree.to_string_lossy(),
2537 &language,
2538 &github_language,
2539 brief.as_deref(),
2540 &attachments,
2541 ),
2542 cwd: worktree,
2543 timeout,
2544 allow_write: true,
2545 sessions,
2546 artifacts: artifacts.clone(),
2547 stem: format!("impl-{label}"),
2548 handover: None,
2549 });
2550 }
2551
2552 self.state.event(
2553 "implement",
2554 format!("{} candidates in parallel", jobs.len()),
2555 );
2556 let mut sent = jobs.clone();
2562 let cache = self.state.config.cache_dir();
2563 let ctx = WaveCtx {
2564 carry_seats: false,
2565 pauses: Some((&self.pause, &self.interrupt)),
2566 run: &run_id,
2567 node: "implement",
2568 prompts: &prompts,
2569 cache: cache.as_deref(),
2570 round: None,
2571 };
2572 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2573 if !wave_blocked(&results) {
2574 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2575 .await;
2576 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2577 .await?;
2578 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2579 .await;
2580 }
2581 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2582
2583 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2584 && results
2585 .iter()
2586 .any(|(_, _, out)| FailClass::of(out).is_some());
2587 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2588 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2589 if self
2593 .state
2594 .candidate_preservation
2595 .contains_key(&self.state.candidates[i].label)
2596 {
2597 self.state.seats.insert(seat.key.clone(), seat);
2598 }
2599 continue;
2600 }
2601 let seat_key = seat.key.clone();
2602 let agent = seat.agent.clone();
2611 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2612 self.state.seats.insert(seat.key.clone(), seat);
2613 let label = self.state.candidates[i].label;
2614 self.state.interrupted_implementations.remove(&label);
2615
2616 let (summary, duration, failed, verified_claim) = match out {
2617 AgentOutcome::Ok(o) => {
2618 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2619 let failed = (!o.usable()).then(|| {
2620 if o.timed_out {
2621 "agent timed out".to_owned()
2622 } else {
2623 format!(
2624 "agent exited with {}",
2625 crate::notices::exit_status(o.exit_code)
2626 )
2627 }
2628 });
2629 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2630 (text, o.duration_ms, failed, verified_claim)
2631 }
2632 AgentOutcome::Dropped(o) => {
2638 let why = o
2639 .dropped
2640 .as_ref()
2641 .map(|d| d.why.as_str())
2642 .unwrap_or("the CLI ended the stream without delivering its answer");
2643 (
2644 String::new(),
2645 o.duration_ms,
2646 Some(format!("the CLI dropped the stream ({why})")),
2647 None,
2648 )
2649 }
2650 AgentOutcome::Quota(o) => {
2651 self.state.quota.push(QuotaLoss {
2652 seat: seat_key,
2653 node: "implement".to_owned(),
2654 at: Timestamp::now(),
2655 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2656 });
2657 (
2658 String::new(),
2659 o.duration_ms,
2660 Some("rate limited (quota); produced no change".to_owned()),
2661 None,
2662 )
2663 }
2664 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2665 (String::new(), 0, Some(e), None)
2666 }
2667 };
2668
2669 let c = &mut self.state.candidates[i];
2672 if !exhausted_the_fallback_chain {
2673 c.agent = agent;
2674 }
2675 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2676 c.duration_ms = duration;
2677 c.failed = failed;
2678 c.verified_noop = verified_claim;
2679 self.state
2680 .candidate_preservation
2681 .insert(label, "awaiting persistence".to_owned());
2682 }
2683 self.state.save()?;
2684 self.persist_candidates().await?;
2685
2686 if resource_blocked {
2687 return self.stop_for_resource();
2688 }
2689 if selected_failed {
2690 self.state.status = RunStatus::Blocked;
2691 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2692 self.state.save()?;
2693 bail!(
2694 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2695 );
2696 }
2697 self.after_implement()
2698 }
2699
2700 async fn persist_candidates(&mut self) -> Result<()> {
2702 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2703 for label in labels {
2704 let i = self
2705 .state
2706 .candidates
2707 .iter()
2708 .position(|c| c.label == label)
2709 .context("pending candidate is missing")?;
2710 let c = self.state.candidates[i].clone();
2711 let interrupted = self.state.interrupted_implementations.contains(&label);
2712 let result = async {
2713 let wait = crate::repo_lock::WaitPolicy::new(
2714 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2715 || false,
2716 );
2717 let _coordination =
2718 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2719 if let Err(e) = git::snapshot_uncommitted(
2722 &c.worktree,
2723 &self
2724 .state
2725 .dir()
2726 .join("artifacts")
2727 .join(format!("candidate-{label}-recovery")),
2728 )
2729 .await
2730 {
2731 self.state.event(
2732 "preserve",
2733 format!("candidate {label} snapshot incomplete: {e:#}"),
2734 );
2735 }
2736 let rescued = git::rescue_commit(
2737 &c.worktree,
2738 &format!("magi: candidate {label} (uncommitted work)"),
2739 )
2740 .await?;
2741 self.state.note_withheld("implement", &rescued.withheld);
2742 let base = &self.state.base_commit;
2743 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2744 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2745 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2746 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2747 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2748 if c.empty && commits > 0 && !patch.trim().is_empty()
2749 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2750 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2751 || self.state.tally.is_some())
2752 {
2753 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2757 }
2758 let c = &mut self.state.candidates[i];
2759 c.commits = commits;
2760 c.stat = stat;
2761 c.files = files;
2762 let no_changes = commits == 0 || patch.trim().is_empty();
2763 c.empty = no_changes && !interrupted;
2764 if interrupted && no_changes {
2765 c.failed = None;
2766 c.verified_noop = None;
2767 }
2768 if !c.empty {
2769 c.failed = None;
2770 c.verified_noop = None;
2771 }
2772 self.state.event(
2773 "preserve",
2774 format!("candidate {label}: {files} files, {commits} commits"),
2775 );
2776 Ok::<_, anyhow::Error>(())
2777 }
2778 .await;
2779 match result {
2780 Ok(()) => {
2781 self.state.candidate_preservation.remove(&label);
2782 if !interrupted || self.state.candidates[i].files > 0 {
2783 self.state.interrupted_implementations.remove(&label);
2784 }
2785 }
2786 Err(e) => {
2787 let reason = format!("candidate {label} preservation failed: {e:#}");
2788 self.state.event("preserve", &reason);
2789 self.state.candidate_preservation.insert(label, reason);
2790 }
2791 }
2792 if self.state.candidate_preservation.is_empty()
2793 && self.state.status == RunStatus::Blocked
2794 && self.state.candidates.iter().all(|c| c.failed.is_none())
2795 {
2796 self.state.status = RunStatus::Implementing;
2800 self.state.event(
2801 "preserve",
2802 "candidate persistence recovered; implementation can continue",
2803 );
2804 }
2805 self.state.save()?;
2806 }
2807 if !self.state.candidate_preservation.is_empty() {
2808 self.state.status = RunStatus::Blocked;
2809 self.state.save()?;
2810 bail!(
2811 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2812 );
2813 }
2814 Ok(())
2815 }
2816
2817 async fn resume_undelivered(
2845 &mut self,
2846 results: &mut [(usize, SeatState, AgentOutcome)],
2847 sent: &[SeatJob],
2848 prompts: &Prompts,
2849 run_id: &str,
2850 ) {
2851 for (wi, seat, out) in results.iter_mut() {
2852 let Some(dropped) = (match &*out {
2853 AgentOutcome::Dropped(o) => o.dropped.clone(),
2854 _ => None,
2855 }) else {
2856 continue;
2857 };
2858 let Some(job) = sent.get(*wi) else { continue };
2859 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2861 self.state.event(
2862 "implement",
2863 format!(
2864 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2865 work is in the tree",
2866 seat.key, dropped.output_tokens, dropped.why
2867 ),
2868 );
2869 continue;
2870 }
2871 if !has_context(&job.spec, seat, job.sessions) {
2879 self.state.event(
2880 "implement",
2881 format!(
2882 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2883 is no session left to resume",
2884 seat.key, dropped.output_tokens, dropped.why
2885 ),
2886 );
2887 continue;
2888 }
2889 self.state.event(
2890 "implement",
2891 format!(
2892 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2893 conversation",
2894 seat.key, dropped.output_tokens, dropped.why
2895 ),
2896 );
2897 let mut retry = job.clone();
2898 retry.seat = seat.clone();
2899 retry.prompt = prompt::resume_after_drop(&dropped.why);
2900 retry.timeout = retry_budget(job.timeout, true);
2901 retry.stem = format!("{}-resume", job.stem);
2902 let cache = self.state.config.cache_dir();
2903 let ctx = WaveCtx {
2904 carry_seats: false,
2905 pauses: Some((&self.pause, &self.interrupt)),
2906 run: run_id,
2907 node: "implement",
2908 prompts,
2909 cache: cache.as_deref(),
2910 round: None,
2911 };
2912 let (resumed_seat, resumed) =
2913 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2914 *seat = resumed_seat;
2915 *out = resumed;
2916 }
2917 }
2918
2919 async fn resume_seat_handovers(
2984 &mut self,
2985 results: &mut [(usize, SeatState, AgentOutcome)],
2986 sent: &mut [SeatJob],
2987 prompts: &Prompts,
2988 run_id: &str,
2989 ) -> Result<()> {
2990 let instruction = seeded_instruction(&self.state);
2991 let language = self.state.config.graph.language.clone();
2992 let github_language = self.state.config.graph.github_language.clone();
2993 let brief = self
2994 .state
2995 .advice
2996 .as_ref()
2997 .and_then(|a| a.synthesis.as_deref())
2998 .map(str::to_owned);
2999 let attachments = self.state.attachments.clone();
3000 for (wi, seat, out) in results.iter_mut() {
3001 let others: BTreeSet<String> = sent
3004 .iter()
3005 .enumerate()
3006 .filter(|(j, _)| j != wi)
3007 .map(|(_, j)| j.spec.id.clone())
3008 .collect();
3009 let Some(job) = sent.get_mut(*wi) else {
3010 continue;
3011 };
3012 let start = self
3017 .roles
3018 .implementer_roster
3019 .iter()
3020 .position(|s| s.id == job.spec.id)
3021 .unwrap_or(0);
3022 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3023 let mut fallback_attempt = 0usize;
3024 let mut prev: Option<FailClass> = None;
3025 while let Some(cur) = FailClass::of(&*out) {
3026 if !should_hand_over(prev.as_ref(), &cur) {
3027 break;
3028 }
3029 let Some(next) =
3030 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3031 .cloned()
3032 else {
3033 break;
3034 };
3035 tried.insert(next.id.clone());
3036 fallback_attempt += 1;
3037
3038 let coordination = self.coordinate(&job.cwd).await?;
3039 if let Ok(r) = git::rescue_commit(
3040 &job.cwd,
3041 &format!(
3042 "magi: candidate {} (uncommitted work before {} fallback)",
3043 seat.key,
3044 if cur == FailClass::Quota {
3045 "quota"
3046 } else {
3047 "handover"
3048 }
3049 ),
3050 )
3051 .await
3052 {
3053 self.state.note_withheld("implement", &r.withheld);
3054 }
3055 drop(coordination);
3056
3057 record_handover(
3058 &mut self.state,
3059 "implement",
3060 &seat.key,
3061 &seat.agent,
3062 &next.id,
3063 &cur,
3064 &fail_reason(&*out),
3065 );
3066 prev = Some(cur.clone());
3067
3068 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3069 self.state.seats.insert(seat.key.clone(), new_seat.clone());
3070 job.spec = next.clone();
3078 let mut retry = job.clone();
3079 retry.seat = new_seat;
3080 retry.prompt = prompt::implement_in(
3081 &instruction,
3082 &job.cwd.to_string_lossy(),
3083 &language,
3084 &github_language,
3085 brief.as_deref(),
3086 &attachments,
3087 );
3088 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3089 let cache = self.state.config.cache_dir();
3090 let ctx = WaveCtx {
3091 carry_seats: false,
3092 pauses: Some((&self.pause, &self.interrupt)),
3093 run: run_id,
3094 node: "implement",
3095 prompts,
3096 cache: cache.as_deref(),
3097 round: None,
3098 };
3099 let (fallback_seat, fallback_out) = run_one(
3100 retry,
3101 Arc::clone(&self.sem),
3102 &ctx,
3103 &mut self.state,
3104 fallback_attempt,
3105 )
3106 .await;
3107 *seat = fallback_seat;
3108 *out = fallback_out;
3109 }
3110 }
3111 Ok(())
3112 }
3113
3114 async fn resume_unconfirmed_commands(
3138 &mut self,
3139 results: &mut [(usize, SeatState, AgentOutcome)],
3140 sent: &[SeatJob],
3141 prompts: &Prompts,
3142 run_id: &str,
3143 ) {
3144 for (wi, seat, out) in results.iter_mut() {
3145 let AgentOutcome::Ok(o) = &*out else {
3146 continue;
3147 };
3148 if !has_unconfirmed_command(&o.commands) {
3149 continue;
3150 }
3151 let Some(job) = sent.get(*wi) else { continue };
3152 if !has_context(&job.spec, seat, job.sessions) {
3153 self.state.event(
3154 "implement",
3155 format!(
3156 "{}: the reply named a command whose own CLI never confirmed the exit \
3157 status of, but there is no session left to resume",
3158 seat.key
3159 ),
3160 );
3161 continue;
3162 }
3163 self.state.event(
3164 "implement",
3165 format!(
3166 "{}: the reply named a command whose own CLI never confirmed the exit \
3167 status of; resuming the conversation",
3168 seat.key
3169 ),
3170 );
3171 let mut retry = job.clone();
3172 retry.seat = seat.clone();
3173 retry.prompt = prompt::resume_incomplete(
3174 "a command in your last reply had no confirmed exit status",
3175 );
3176 retry.timeout = retry_budget(job.timeout, true);
3177 retry.stem = format!("{}-confirm", job.stem);
3178 let cache = self.state.config.cache_dir();
3179 let ctx = WaveCtx {
3180 carry_seats: false,
3181 pauses: Some((&self.pause, &self.interrupt)),
3182 run: run_id,
3183 node: "implement",
3184 prompts,
3185 cache: cache.as_deref(),
3186 round: None,
3187 };
3188 let (resumed_seat, resumed) =
3189 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3190 *seat = resumed_seat;
3191 *out = resumed;
3192 }
3193 }
3194
3195 async fn continue_fix_report(
3216 &mut self,
3217 mut seat: SeatState,
3218 parse_err: String,
3219 job: &SeatJob,
3220 prompts: &Prompts,
3221 run_id: &str,
3222 round: usize,
3223 ) -> (
3224 SeatState,
3225 Option<FixReport>,
3226 Option<String>,
3227 ContinuationRecord,
3228 ) {
3229 let mut last_err = parse_err;
3230 let mut cumulative_wait_ms = 0u64;
3231 let mut attempts = 0usize;
3232 loop {
3233 if !has_context(&job.spec, &seat, job.sessions) {
3234 self.state.event(
3235 "fix",
3236 format!(
3237 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3238 session left to resume into"
3239 ),
3240 );
3241 let outcome = if attempts == 0 {
3242 ContinuationOutcome::NoSession
3243 } else {
3244 ContinuationOutcome::Exhausted
3245 };
3246 return (
3247 seat,
3248 None,
3249 Some(format!("unparsable fix report: {last_err}")),
3250 ContinuationRecord {
3251 attempts,
3252 cumulative_wait_ms,
3253 outcome,
3254 },
3255 );
3256 }
3257 if attempts >= MAX_FIX_CONTINUATIONS {
3258 self.state.event(
3259 "fix",
3260 format!(
3261 "round {round}: fixer's reply still had no adoption report after \
3262 {attempts} continuation(s) ({last_err}); giving up"
3263 ),
3264 );
3265 return (
3266 seat,
3267 None,
3268 Some(format!(
3269 "unparsable fix report after {attempts} continuation(s): {last_err}"
3270 )),
3271 ContinuationRecord {
3272 attempts,
3273 cumulative_wait_ms,
3274 outcome: ContinuationOutcome::Exhausted,
3275 },
3276 );
3277 }
3278 attempts += 1;
3279 self.state.event(
3280 "fix",
3281 format!(
3282 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3283 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3284 ),
3285 );
3286 let mut retry = job.clone();
3287 retry.seat = seat.clone();
3288 retry.prompt = prompt::resume_incomplete(&last_err);
3289 retry.timeout = retry_budget(job.timeout, true);
3290 retry.stem = format!("{}-continue{attempts}", job.stem);
3291 let cache = self.state.config.cache_dir();
3292 let ctx = WaveCtx {
3293 carry_seats: false,
3294 pauses: Some((&self.pause, &self.interrupt)),
3295 run: run_id,
3296 node: "fix",
3297 prompts,
3298 cache: cache.as_deref(),
3299 round: Some(round),
3300 };
3301 let (resumed_seat, resumed_out) = run_one(
3302 retry,
3303 Arc::clone(&self.sem),
3304 &ctx,
3305 &mut self.state,
3306 attempts,
3307 )
3308 .await;
3309 seat = resumed_seat;
3310 match resumed_out {
3311 AgentOutcome::Ok(o) => {
3312 cumulative_wait_ms += o.duration_ms;
3313 match verdict::extract_json::<FixReport>(&o.text) {
3314 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3315 self.state.event(
3316 "fix",
3317 format!(
3318 "round {round}: fixer's adoption report recovered after \
3319 {attempts} continuation(s)"
3320 ),
3321 );
3322 return (
3323 seat,
3324 Some(report),
3325 None,
3326 ContinuationRecord {
3327 attempts,
3328 cumulative_wait_ms,
3329 outcome: ContinuationOutcome::Resumed,
3330 },
3331 );
3332 }
3333 Ok(_) => {
3341 last_err = "the reply parsed, but it reported a command whose own CLI \
3342 never confirmed an exit status"
3343 .to_owned();
3344 }
3345 Err(e) => last_err = e.to_string(),
3346 }
3347 }
3348 AgentOutcome::Quota(o) => {
3349 cumulative_wait_ms += o.duration_ms;
3350 self.state.quota.push(QuotaLoss {
3351 seat: seat.key.clone(),
3352 node: "fix".to_owned(),
3353 at: Timestamp::now(),
3354 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3355 });
3356 self.state.event(
3357 "fix",
3358 format!(
3359 "round {round}: continuation rate limited (quota); not retrying now"
3360 ),
3361 );
3362 return (
3363 seat,
3364 None,
3365 Some("rate limited (quota) while recovering the fix report".to_owned()),
3366 ContinuationRecord {
3367 attempts,
3368 cumulative_wait_ms,
3369 outcome: ContinuationOutcome::QuotaLost,
3370 },
3371 );
3372 }
3373 AgentOutcome::Dropped(o) => {
3374 cumulative_wait_ms += o.duration_ms;
3375 let why = o
3376 .dropped
3377 .as_ref()
3378 .map(|d| d.why.as_str())
3379 .unwrap_or("the CLI ended the stream without delivering its answer");
3380 last_err = format!("the CLI dropped the stream ({why})");
3381 }
3382 AgentOutcome::Failed(e) => last_err = e,
3383 AgentOutcome::ResourceBlocked(e) => {
3386 return (
3387 seat,
3388 None,
3389 Some(format!("the continuation was not started: {e}")),
3390 ContinuationRecord {
3391 attempts: attempts.saturating_sub(1),
3392 cumulative_wait_ms,
3393 outcome: ContinuationOutcome::NoSession,
3394 },
3395 );
3396 }
3397 }
3398 }
3399 }
3400
3401 fn after_implement(&mut self) -> Result<()> {
3402 if !self.state.candidate_preservation.is_empty() {
3403 bail!("candidate preservation incomplete; judging is unsafe");
3404 }
3405 if self.state.leaks.is_empty() {
3407 let cfg = self.state.config.blind.clone();
3408 let mut leaks = Vec::new();
3409 for c in &self.state.candidates {
3410 let Some(patch) =
3411 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3412 else {
3413 continue;
3414 };
3415 leaks.extend(blind::scan(
3416 &format!("candidate {} patch", c.label),
3417 &patch,
3418 &cfg.vendor_tokens,
3419 ));
3420 }
3421 if !leaks.is_empty() {
3422 let summary = leaks
3423 .iter()
3424 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3425 .collect::<Vec<_>>()
3426 .join(", ");
3427 match cfg.on_leak {
3428 LeakPolicy::Fail => {
3429 self.state.status = RunStatus::Failed;
3430 self.state
3431 .event("blind", format!("vendor text in a patch: {summary}"));
3432 self.state.leaks = leaks;
3433 self.state.save()?;
3434 self.settle_questions();
3435 bail!(
3436 "blind.on_leak = \"fail\" and vendor text reached a \
3437 judged patch: {summary}"
3438 );
3439 }
3440 LeakPolicy::Redact => self.state.event(
3441 "blind",
3442 format!("redacting vendor text for judging: {summary}"),
3443 ),
3444 LeakPolicy::Warn => self.state.event(
3445 "blind",
3446 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3447 ),
3448 }
3449 self.state.leaks = leaks;
3450 }
3451 }
3452
3453 if self.state.viable().is_empty() {
3454 if self.state.all_candidates_verified_noop() {
3455 self.state.status = RunStatus::VerifiedNoop;
3466 self.state.save()?;
3467 self.settle_questions();
3468 return Ok(());
3469 }
3470 self.state.status = RunStatus::Failed;
3471 self.state.save()?;
3472 self.settle_questions();
3473 bail!("no candidate produced a change; nothing to judge");
3474 }
3475 self.state.status = RunStatus::Judging;
3476 self.state.save()?;
3477 Ok(())
3478 }
3479
3480 async fn judge(&mut self) -> Result<()> {
3483 let run_id = self.state.id.clone();
3488 let prompts = self.state.config.prompts.clone();
3489 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3490 return Ok(());
3491 }
3492 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3493 if viable.len() == 1 {
3494 self.state.judge_skipped = true;
3501 self.state.event(
3502 "judge",
3503 format!(
3504 "only candidate {} produced a change; judging skipped",
3505 viable[0].label
3506 ),
3507 );
3508 self.state.save()?;
3509 return Ok(());
3510 }
3511 self.state.status = RunStatus::Judging;
3512
3513 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3514 let language = self.state.config.graph.language.clone();
3515 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3516 let sessions = self.state.config.graph.sessions;
3517 let artifacts = agent::artifacts_dir(&self.state.dir());
3518 let root = self.state.worktree_root();
3519 let base_short = short(&self.state.base_commit);
3520
3521 let mut jobs = Vec::new();
3522 let mut orders = Vec::new();
3523 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3524 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3525 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3526 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3527 let seat_key = format!("judge-{}", j + 1);
3528 let seat = self.seat(&seat_key, &spec.id);
3529 jobs.push(SeatJob {
3530 prompt: prompt::judge(
3531 &self.state.instruction,
3532 &views,
3533 self.roles.judges.len(),
3534 &base_short,
3535 &language,
3536 ),
3537 spec,
3538 seat,
3539 cwd: root.join(format!("judge-{}", j + 1)),
3540 timeout,
3541 allow_write: false,
3542 sessions,
3543 artifacts: artifacts.clone(),
3544 stem: format!("judge-{}", j + 1),
3545 handover: None,
3546 });
3547 }
3548
3549 self.state.event(
3550 "judge",
3551 format!(
3552 "{} judges ranking {} candidates blind",
3553 jobs.len(),
3554 viable.len()
3555 ),
3556 );
3557 let labels_for_check = labels.clone();
3558 let mut quota_losses = Vec::new();
3559 let cache = self.state.config.cache_dir();
3560 let ctx = WaveCtx {
3561 carry_seats: false,
3562 pauses: Some((&self.pause, &self.interrupt)),
3563 run: &run_id,
3564 node: "judge",
3565 prompts: &prompts,
3566 cache: cache.as_deref(),
3567 round: None,
3568 };
3569 let results = ask_json_wave::<Ranking>(
3570 jobs,
3571 Arc::clone(&self.sem),
3572 self.state.config.graph.retries,
3573 &self.roles.judge_roster,
3574 &ctx,
3575 &mut quota_losses,
3576 &mut self.state,
3577 &move |r: &Ranking| r.validate(&labels_for_check),
3578 )
3579 .await;
3580 if self.state.resource_block.is_some() {
3581 return self.stop_for_resource();
3582 }
3583 self.state.quota.extend(quota_losses);
3584
3585 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3586 let agent_id = seat.agent.clone();
3587 self.state.seats.insert(seat.key.clone(), seat);
3588 let mut record = Judgement {
3589 judge: j + 1,
3590 seat: format!("judge-{}", j + 1),
3591 agent: agent_id,
3592 ranking: Vec::new(),
3593 reasons: BTreeMap::new(),
3594 confidence: None,
3595 strengths: BTreeMap::new(),
3596 order: orders[j].clone(),
3597 failed: None,
3598 duration_ms: 0,
3599 };
3600 match res {
3601 Ok((ranking, out)) => {
3602 record.ranking = ranking.normalized();
3603 record.reasons = ranking.reasons;
3604 record.strengths = ranking.strengths;
3605 record.confidence = ranking.confidence;
3606 record.duration_ms = out.duration_ms;
3607 self.state.event(
3608 "judge",
3609 format!(
3610 "judge {} ranked {}",
3611 j + 1,
3612 record.ranking.iter().collect::<String>()
3613 ),
3614 );
3615 }
3616 Err(e) => {
3617 record.failed = Some(e.to_string());
3618 self.state
3619 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3620 }
3621 }
3622 self.state.judgements.push(record);
3623 self.state.save()?;
3624 }
3625 Ok(())
3626 }
3627
3628 async fn deliberate(&mut self) -> Result<()> {
3631 if self.hold_base_sync().await? {
3632 return Ok(());
3633 }
3634 let run_id = self.state.id.clone();
3639 let prompts = self.state.config.prompts.clone();
3640 if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3641 return Ok(());
3642 }
3643 let tops: Vec<char> = self
3644 .state
3645 .judgements
3646 .iter()
3647 .filter_map(|j| j.ranking.first().copied())
3648 .collect();
3649 let rounds = self.state.config.graph.deliberate_rounds;
3650 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3651 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3652 self.state.event(
3653 "deliberate",
3654 format!("judges agreed on {} outright; no deliberation", tops[0]),
3655 );
3656 }
3657 self.state.status = RunStatus::Voting;
3658 self.state.save()?;
3659 return Ok(());
3660 }
3661
3662 self.state.status = RunStatus::Deliberating;
3663 self.state.event(
3664 "deliberate",
3665 format!(
3666 "split: first choices were {} — opening {rounds} round(s)",
3667 tops.iter().collect::<String>()
3668 ),
3669 );
3670
3671 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3672 let language = self.state.config.graph.language.clone();
3673 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3674 let sessions = self.state.config.graph.sessions;
3675 let artifacts = agent::artifacts_dir(&self.state.dir());
3676 let root = self.state.worktree_root();
3677 let base_short = short(&self.state.base_commit);
3678
3679 for round in 1..=rounds {
3683 let mut turns: Vec<DeliberationTurn> = Vec::new();
3684 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3685 if self.state.judgements[j].failed.is_some() {
3686 continue;
3687 }
3688 let seat_key = format!("judge-{}", j + 1);
3689 let spec = self.occupant(&seat_key, spec);
3690 let mut seat = self.seat(&seat_key, &spec.id);
3691 let transcript = self.transcript(&turns, j);
3692 let build = |context: Option<&str>| {
3693 prompt::deliberate(
3694 &self.state.instruction,
3695 context,
3696 &transcript,
3697 round,
3698 rounds,
3699 &language,
3700 )
3701 };
3702 let block = self.candidate_block(&viable, &base_short);
3703 let full = build(Some(&block));
3704 let text = if has_context(&spec, &seat, sessions) {
3705 build(None)
3706 } else {
3707 full.clone()
3708 };
3709 let job = SeatJob {
3710 spec,
3711 seat: seat.clone(),
3712 prompt: text,
3713 cwd: root.join(format!("judge-{}", j + 1)),
3714 timeout,
3715 allow_write: false,
3716 sessions,
3717 artifacts: artifacts.clone(),
3718 stem: format!("delib-{round}-judge-{}", j + 1),
3719 handover: Some(full),
3720 };
3721 let cache = self.state.config.cache_dir();
3722 let ctx = WaveCtx {
3723 carry_seats: false,
3724 pauses: Some((&self.pause, &self.interrupt)),
3725 run: &run_id,
3726 node: "deliberate",
3727 prompts: &prompts,
3728 cache: cache.as_deref(),
3729 round: None,
3730 };
3731 let mut losses = Vec::new();
3735 let mut results = ask_wave_with::<String>(
3736 vec![job],
3737 Arc::clone(&self.sem),
3738 0,
3739 &self.roles.judge_roster,
3740 &ctx,
3741 &mut losses,
3742 &mut self.state,
3743 &|text: &str| {
3744 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3745 },
3746 )
3747 .await;
3748 self.state.quota.extend(losses);
3749 let (updated, res, _) = results.pop().expect("one job in, one result out");
3750 seat = updated;
3751 let agent_id = seat.agent.clone();
3752 self.state.seats.insert(seat.key.clone(), seat);
3753 let body = match res {
3754 Ok((body, _)) => body,
3755 Err(e) => {
3758 self.state
3759 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3760 continue;
3761 }
3762 };
3763 let tentative = verdict::extract_json::<Position>(&body)
3764 .ok()
3765 .and_then(|p| p.tentative)
3766 .and_then(|s| s.trim().chars().next())
3767 .map(|c| c.to_ascii_uppercase());
3768 self.state.event(
3769 "deliberate",
3770 format!(
3771 "round {round}: judge {} now favours {}",
3772 j + 1,
3773 tentative.map_or("—".to_owned(), |c| c.to_string())
3774 ),
3775 );
3776 turns.push(DeliberationTurn {
3777 judge: j + 1,
3778 agent: agent_id,
3779 body: blind::sanitize_prose(&body, &self.state.config.blind),
3780 tentative,
3781 });
3782 }
3783 self.state
3784 .deliberation
3785 .push(DeliberationRound { round, turns });
3786 self.state.save()?;
3787 }
3788
3789 self.state.status = RunStatus::Voting;
3790 self.state.save()?;
3791 Ok(())
3792 }
3793
3794 async fn vote(&mut self) -> Result<()> {
3797 if self.hold_base_sync().await? {
3798 return Ok(());
3799 }
3800 let run_id = self.state.id.clone();
3805 let prompts = self.state.config.prompts.clone();
3806 if !self.state.votes.is_empty() || self.state.tally.is_some() {
3807 return Ok(());
3808 }
3809 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3810 if viable.len() == 1 {
3811 return Ok(());
3812 }
3813 self.state.status = RunStatus::Voting;
3814
3815 let language = self.state.config.graph.language.clone();
3816 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3817 let sessions = self.state.config.graph.sessions;
3818 let artifacts = agent::artifacts_dir(&self.state.dir());
3819 let root = self.state.worktree_root();
3820 let base_short = short(&self.state.base_commit);
3821 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3822
3823 let mut jobs = Vec::new();
3824 let mut seats_at = Vec::new();
3825 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3826 if self
3827 .state
3828 .judgements
3829 .get(j)
3830 .is_some_and(|r| r.failed.is_some())
3831 {
3832 continue;
3833 }
3834 let seat_key = format!("judge-{}", j + 1);
3835 let spec = self.occupant(&seat_key, spec);
3836 let seat = self.seat(&seat_key, &spec.id);
3837 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3838 let text = if has_context(&spec, &seat, sessions) {
3839 prompt::final_vote(&viable, &language)
3840 } else {
3841 full.clone()
3842 };
3843 jobs.push(SeatJob {
3844 spec,
3845 seat,
3846 prompt: text,
3847 cwd: root.join(format!("judge-{}", j + 1)),
3848 timeout,
3849 allow_write: false,
3850 sessions,
3851 artifacts: artifacts.clone(),
3852 stem: format!("vote-judge-{}", j + 1),
3853 handover: Some(full),
3854 });
3855 seats_at.push(j);
3856 }
3857
3858 self.state.event(
3859 "vote",
3860 format!(
3861 "collecting {} final votes one by one, privately",
3862 jobs.len()
3863 ),
3864 );
3865 let allowed = viable.clone();
3866 let mut quota_losses = Vec::new();
3867 let cache = self.state.config.cache_dir();
3868 let ctx = WaveCtx {
3869 carry_seats: false,
3870 pauses: Some((&self.pause, &self.interrupt)),
3871 run: &run_id,
3872 node: "vote",
3873 prompts: &prompts,
3874 cache: cache.as_deref(),
3875 round: None,
3876 };
3877 let results = ask_json_wave::<FinalVote>(
3878 jobs,
3879 Arc::clone(&self.sem),
3880 self.state.config.graph.retries,
3881 &self.roles.judge_roster,
3882 &ctx,
3883 &mut quota_losses,
3884 &mut self.state,
3885 &move |v: &FinalVote| match v.label() {
3886 Some(c) if allowed.contains(&c) => Ok(()),
3887 other => bail!("vote {other:?} is not one of {allowed:?}"),
3888 },
3889 )
3890 .await;
3891 if self.state.resource_block.is_some() {
3892 return self.stop_for_resource();
3893 }
3894 self.state.quota.extend(quota_losses);
3895
3896 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3897 let agent_id = seat.agent.clone();
3898 self.state.seats.insert(seat.key.clone(), seat);
3899 let initial = self
3900 .state
3901 .judgements
3902 .get(j)
3903 .and_then(|r| r.ranking.first().copied());
3904 let mut record = VoteRecord {
3905 judge: j + 1,
3906 agent: agent_id,
3907 vote: None,
3908 reason: String::new(),
3909 changed: false,
3910 };
3911 match res {
3912 Ok((v, _)) => {
3913 record.vote = v.label();
3914 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3915 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3916 self.state.event(
3917 "vote",
3918 format!(
3919 "judge {} voted {}{}",
3920 j + 1,
3921 record.vote.unwrap_or('?'),
3922 if record.changed { " (changed)" } else { "" }
3923 ),
3924 );
3925 }
3926 Err(e) => {
3927 self.state
3928 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3929 }
3930 }
3931 self.state.votes.push(record);
3932 self.state.save()?;
3933 }
3934 Ok(())
3935 }
3936
3937 fn tally(&mut self) -> Result<()> {
3940 if self.state.tally.is_some() {
3941 return Ok(());
3942 }
3943 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3944 let tops: Vec<char> = self
3945 .state
3946 .judgements
3947 .iter()
3948 .filter_map(|j| j.ranking.first().copied())
3949 .collect();
3950 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3951
3952 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3955 let mut cast: Vec<char> = Vec::new();
3956 for (i, j) in self.state.judgements.iter().enumerate() {
3957 let vote = self
3958 .state
3959 .votes
3960 .iter()
3961 .find(|v| v.judge == i + 1)
3962 .and_then(|v| v.vote)
3963 .or_else(|| j.ranking.first().copied());
3964 if let Some(v) = vote {
3965 *first_choice.entry(v).or_insert(0) += 1;
3966 cast.push(v);
3967 }
3968 }
3969
3970 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3971 for j in &self.state.judgements {
3972 let n = j.ranking.len();
3973 for (pos, label) in j.ranking.iter().enumerate() {
3974 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3975 }
3976 }
3977
3978 let best = first_choice.values().copied().max().unwrap_or(0);
3979 let mut leaders: Vec<char> = first_choice
3980 .iter()
3981 .filter(|(_, v)| **v == best)
3982 .map(|(k, _)| *k)
3983 .collect();
3984 let mut tie_break = None;
3985 if leaders.len() > 1 {
3986 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3987 let borda_leaders: Vec<char> = leaders
3988 .iter()
3989 .copied()
3990 .filter(|l| borda[l] == top_borda)
3991 .collect();
3992 tie_break = Some(if borda_leaders.len() == 1 {
3993 format!(
3994 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3995 leaders.len()
3996 )
3997 } else {
3998 format!(
3999 "{} way tie on both first-choice votes and Borda points, broken by label order",
4000 leaders.len()
4001 )
4002 });
4003 leaders = borda_leaders;
4004 leaders.sort_unstable();
4005 }
4006 let winner = *leaders
4007 .first()
4008 .or(viable.first())
4009 .context("no candidate to declare a winner from")?;
4010
4011 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4012 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4013 let deliberated = !self.state.deliberation.is_empty();
4014
4015 let quota_seats: std::collections::BTreeSet<&str> =
4019 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4020 let mut present = 0usize;
4021 for (i, j) in self.state.judgements.iter().enumerate() {
4022 if quota_seats.contains(j.seat.as_str()) {
4023 continue;
4024 }
4025 let ranked = !j.ranking.is_empty() && j.failed.is_none();
4026 let voted = self
4027 .state
4028 .votes
4029 .iter()
4030 .any(|v| v.judge == i + 1 && v.vote.is_some());
4031 if ranked || voted {
4032 present += 1;
4033 }
4034 }
4035 let needs_quorum = viable.len() > 1;
4041 let judges_total = if needs_quorum {
4042 self.roles.judges.len()
4043 } else {
4044 0
4045 };
4046 let quorum = if needs_quorum {
4047 judges_total / 2 + 1
4048 } else {
4049 0
4050 };
4051 let met_quorum = !needs_quorum || present >= quorum;
4052 let uncontested = (!needs_quorum).then(|| {
4053 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4054 });
4055
4056 self.state.event(
4057 "tally",
4058 match &uncontested {
4059 Some(reason) => format!("winner {winner} — {reason}"),
4060 None => format!(
4061 "winner {winner} — votes {} | initial {} | {} changed | \
4062 {present}/{judges_total} judges{}",
4063 first_choice
4064 .iter()
4065 .map(|(k, v)| format!("{k}:{v}"))
4066 .collect::<Vec<_>>()
4067 .join(" "),
4068 if unanimous_initial {
4069 "unanimous"
4070 } else {
4071 "split"
4072 },
4073 changed_votes,
4074 if met_quorum {
4075 String::new()
4076 } else {
4077 format!(" — below quorum ({quorum} required)")
4078 },
4079 ),
4080 },
4081 );
4082 if !met_quorum {
4083 self.state.event(
4084 "stall",
4085 format!(
4086 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4087 the run stops here, resumable"
4088 ),
4089 );
4090 }
4091 self.state.tally = Some(Tally {
4092 first_choice,
4093 borda,
4094 winner,
4095 rankings: tops.len(),
4096 unanimous_initial,
4097 deliberated,
4098 changed_votes,
4099 unanimous_final,
4100 tie_break,
4101 judges: judges_total,
4102 present,
4103 quorum,
4104 met_quorum,
4105 uncontested,
4106 });
4107 self.state.status = if met_quorum {
4108 RunStatus::Reviewing
4109 } else {
4110 RunStatus::Stalled
4111 };
4112 self.state.save()?;
4113 Ok(())
4114 }
4115
4116 #[allow(clippy::too_many_lines)]
4137 async fn recover_stall(&mut self) -> Result<bool> {
4138 let run_id = self.state.id.clone();
4143 let prompts = self.state.config.prompts.clone();
4144 let quota_seats: BTreeSet<&str> =
4149 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4150 let absent: Vec<String> = self
4151 .state
4152 .judgements
4153 .iter()
4154 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4155 .map(|j| j.seat.clone())
4156 .collect();
4157 if absent.is_empty() {
4158 return Ok(false);
4159 }
4160 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4161 if viable.len() <= 1 {
4162 return Ok(false);
4163 }
4164 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4165 let language = self.state.config.graph.language.clone();
4166 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4167 let sessions = self.state.config.graph.sessions;
4168 let artifacts = agent::artifacts_dir(&self.state.dir());
4169 let root = self.state.worktree_root();
4170 let base_short = short(&self.state.base_commit);
4171 let candidates: Vec<Candidate> = viable.clone();
4172
4173 let mut positions: Vec<usize> = absent
4175 .iter()
4176 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4177 .collect();
4178 if positions.is_empty() {
4179 return Ok(false);
4180 }
4181 positions.sort_unstable();
4182 positions.dedup();
4183
4184 let mut judge_jobs = Vec::new();
4186 for &j in &positions {
4187 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4188 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4189 let seat_key = format!("judge-{}", j + 1);
4190 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4191 let seat = self.seat(&seat_key, &spec.id);
4192 judge_jobs.push(SeatJob {
4193 spec,
4194 seat,
4195 prompt: prompt::judge(
4196 &self.state.instruction,
4197 &views,
4198 self.roles.judges.len(),
4199 &base_short,
4200 &language,
4201 ),
4202 cwd: root.join(seat_key),
4203 timeout,
4204 allow_write: false,
4205 sessions,
4206 artifacts: artifacts.clone(),
4207 stem: format!("judge-{}-recover", j + 1),
4208 handover: None,
4209 });
4210 }
4211
4212 let labels_for_check = labels.clone();
4213 let mut judge_losses = Vec::new();
4214 let retries = self.state.config.graph.retries;
4215 let cache = self.state.config.cache_dir();
4216 let ctx = WaveCtx {
4217 carry_seats: false,
4218 pauses: Some((&self.pause, &self.interrupt)),
4219 run: &run_id,
4220 node: "judge",
4221 prompts: &prompts,
4222 cache: cache.as_deref(),
4223 round: None,
4224 };
4225 let results = ask_json_wave::<Ranking>(
4226 judge_jobs,
4227 Arc::clone(&self.sem),
4228 retries,
4229 &self.roles.judge_roster,
4230 &ctx,
4231 &mut judge_losses,
4232 &mut self.state,
4233 &move |r: &Ranking| r.validate(&labels_for_check),
4234 )
4235 .await;
4236 if self.state.resource_block.is_some() {
4237 self.stop_for_resource()?;
4238 return Ok(false);
4239 }
4240
4241 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4243 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4244 let agent_id = seat.agent.clone();
4245 self.state.seats.insert(seat.key.clone(), seat);
4246 let record = &mut self.state.judgements[j];
4247 match res {
4248 Ok((ranking, out)) => {
4249 record.agent = agent_id;
4250 record.ranking = ranking.normalized();
4251 record.reasons = ranking.reasons;
4252 record.strengths = ranking.strengths;
4253 record.confidence = ranking.confidence;
4254 record.failed = None;
4255 record.duration_ms = out.duration_ms;
4256 recovered.insert(j);
4257 self.state.event(
4258 "recover",
4259 format!("judge {} ranked again after the limit", j + 1),
4260 );
4261 }
4262 Err(e) => {
4263 self.state
4264 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4265 }
4266 }
4267 }
4268
4269 let mut vote_jobs = Vec::new();
4271 let mut vote_pos: Vec<usize> = Vec::new();
4272 for &j in &recovered {
4273 let seat_key = format!("judge-{}", j + 1);
4274 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4275 let seat = self.seat(&seat_key, &spec.id);
4276 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4277 let text = if has_context(&spec, &seat, sessions) {
4278 prompt::final_vote(&labels, &language)
4279 } else {
4280 full.clone()
4281 };
4282 vote_jobs.push(SeatJob {
4283 spec,
4284 seat,
4285 prompt: text,
4286 cwd: root.join(seat_key),
4287 timeout,
4288 allow_write: false,
4289 sessions,
4290 artifacts: artifacts.clone(),
4291 stem: format!("vote-judge-{}-recover", j + 1),
4292 handover: Some(full),
4293 });
4294 vote_pos.push(j);
4295 }
4296 let allowed = labels.clone();
4297 let mut vote_losses = Vec::new();
4298 let vote_retries = self.state.config.graph.retries;
4299 let vote_cache = self.state.config.cache_dir();
4300 let ctx = WaveCtx {
4301 carry_seats: false,
4302 pauses: Some((&self.pause, &self.interrupt)),
4303 run: &run_id,
4304 node: "vote",
4305 prompts: &prompts,
4306 cache: vote_cache.as_deref(),
4307 round: None,
4308 };
4309 let votes = ask_json_wave::<FinalVote>(
4310 vote_jobs,
4311 Arc::clone(&self.sem),
4312 vote_retries,
4313 &self.roles.judge_roster,
4314 &ctx,
4315 &mut vote_losses,
4316 &mut self.state,
4317 &move |v: &FinalVote| match v.label() {
4318 Some(c) if allowed.contains(&c) => Ok(()),
4319 other => bail!("vote {other:?} is not one of {allowed:?}"),
4320 },
4321 )
4322 .await;
4323 if self.state.resource_block.is_some() {
4324 self.stop_for_resource()?;
4325 return Ok(false);
4326 }
4327 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4328 let agent_id = seat.agent.clone();
4329 self.state.seats.insert(seat.key.clone(), seat);
4330 match res {
4331 Ok((v, _)) => {
4332 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4333 rec.vote = v.label();
4334 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4335 } else {
4336 self.state.votes.push(VoteRecord {
4337 judge: j + 1,
4338 agent: agent_id,
4339 vote: v.label(),
4340 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4341 changed: false,
4342 });
4343 }
4344 self.state.event(
4345 "recover",
4346 format!("judge {} voted again after the limit", j + 1),
4347 );
4348 }
4349 Err(e) => {
4350 self.state
4351 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4352 }
4353 }
4354 }
4355
4356 let recovered_keys: BTreeSet<String> = recovered
4360 .iter()
4361 .map(|&j| format!("judge-{}", j + 1))
4362 .collect();
4363 self.state
4364 .quota
4365 .retain(|q| !recovered_keys.contains(&q.seat));
4366 for loss in judge_losses.into_iter().chain(vote_losses) {
4370 if recovered_keys.contains(&loss.seat) {
4371 continue;
4372 }
4373 self.state.quota.retain(|q| q.seat != loss.seat);
4374 self.state.quota.push(loss);
4375 }
4376
4377 self.state.tally = None;
4379 self.tally()?;
4380 Ok(self
4381 .state
4382 .tally
4383 .as_ref()
4384 .map(|t| t.met_quorum)
4385 .unwrap_or(false))
4386 }
4387
4388 async fn graft(&mut self) -> Result<()> {
4404 let Err(e) = self.graft_inner().await else {
4405 return Ok(());
4406 };
4407 if e.is::<crate::repo_lock::ResourceBlocked>() {
4408 return Err(e);
4409 }
4410 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4411 self.state.status = RunStatus::Blocked;
4412 self.state.event("graft", format!("blocked: {e:#}"));
4413 self.state.save()?;
4414 return Err(e);
4415 }
4416 if let Some(rec) = self.state.graft.clone()
4421 && !rec.outcome.settled()
4422 {
4423 if let Some(winner) = self.state.winner().cloned() {
4424 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4425 if re.is::<crate::repo_lock::ResourceBlocked>() {
4426 return Err(re);
4427 }
4428 self.state.status = RunStatus::Blocked;
4429 self.state.event("graft", format!("blocked: {re:#}"));
4430 self.state.save()?;
4431 return Err(re);
4432 }
4433 }
4434 if let Some(g) = self.state.graft.as_mut() {
4435 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4436 }
4437 }
4438 self.state.event("graft", format!("skipped: {e:#}"));
4439 if let Err(e) = self.state.save() {
4440 tracing::warn!("save after graft: {e:#}");
4441 }
4442 Ok(())
4443 }
4444
4445 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4448 let _coordination = self.coordinate(wt).await?;
4449 crate::recovery::restore_repair(wt, pre)
4450 .await
4451 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4452 }
4453
4454 fn graft_skipped(&mut self, why: &str) {
4456 let message = format!("skipped: {why}");
4457 if !self
4458 .state
4459 .events
4460 .iter()
4461 .any(|e| e.node == "graft" && e.message == message)
4462 {
4463 self.state.event("graft", message);
4464 }
4465 }
4466
4467 async fn graft_inner(&mut self) -> Result<()> {
4468 if let Some(rec) = self.state.graft.clone()
4471 && !rec.outcome.settled()
4472 {
4473 if self.state.reviews.is_empty()
4477 && let Some(winner) = self.state.winner().cloned()
4478 {
4479 self.graft_restore(&winner.worktree, &rec.pre).await?;
4480 }
4481 if rec.outcome == GraftOutcome::WaitingForRestore {
4482 self.state.graft = None;
4483 self.state.save()?;
4484 } else {
4485 if let Some(g) = self.state.graft.as_mut() {
4486 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4487 }
4488 self.state.event(
4489 "graft",
4490 "rolled back: the previous round was interrupted; the winner is untouched",
4491 );
4492 self.state.save()?;
4493 return Ok(());
4494 }
4495 }
4496 let plan = match graft::plan(&self.state) {
4497 Ok(plan) => plan,
4498 Err(skip) => {
4499 if skip != graft::Skip::AlreadyDone {
4500 self.graft_skipped(skip.reason());
4501 self.state.save()?;
4502 }
4503 return Ok(());
4504 }
4505 };
4506 let Some(winner) = self.state.winner().cloned() else {
4507 return Ok(());
4508 };
4509 let wt = winner.worktree.clone();
4510 if !git::is_clean(&wt).await? {
4511 self.graft_skipped("the winner's worktree has uncommitted changes");
4512 self.state.save()?;
4513 return Ok(());
4514 }
4515 let pre = git::rev_parse(&wt, "HEAD").await?;
4516 self.state.graft = Some(GraftRecord {
4519 pre: pre.clone(),
4520 post: None,
4521 lines: None,
4522 limit: None,
4523 agent: None,
4524 outcome: GraftOutcome::Running,
4525 requested: graft::requested(&plan),
4526 adoption: None,
4527 adoption_error: None,
4528 });
4529 self.state.event(
4530 "graft",
4531 format!(
4532 "porting strengths of candidate(s) {} onto {}",
4533 plan.strengths
4534 .iter()
4535 .map(|(l, _, _)| l.to_string())
4536 .collect::<Vec<_>>()
4537 .join(", "),
4538 plan.winner
4539 ),
4540 );
4541 self.state.save()?;
4542
4543 let fix_prompt = prompt::graft(
4544 &self.state.instruction,
4545 plan.winner,
4546 &plan.strengths,
4547 &self.state.config.graph.language,
4548 );
4549 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4550 let sessions = self.state.config.graph.sessions;
4551 let artifacts = agent::artifacts_dir(&self.state.dir());
4552 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4553 let (_, seat, out) = self
4554 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4555 SeatJob {
4556 prompt: fix_prompt.clone(),
4557 spec,
4558 seat,
4559 cwd: wt.clone(),
4560 timeout,
4561 allow_write: true,
4562 sessions,
4563 artifacts: artifacts.clone(),
4564 stem: "graft".to_owned(),
4565 handover: None,
4566 }
4567 })
4568 .await;
4569 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4570 self.state.graft.as_mut().expect("graft record").outcome =
4573 GraftOutcome::WaitingForRestore;
4574 return self.stop_for_resource();
4575 }
4576 let mut why: Option<String> = None;
4577 match &out {
4578 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4579 why = Some("the fixer produced nothing usable".to_owned());
4580 }
4581 AgentOutcome::Ok(_) => {}
4582 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4583 AgentOutcome::Quota(o) => {
4584 self.state.quota.push(QuotaLoss {
4585 seat: seat.key.clone(),
4586 node: "graft".to_owned(),
4587 at: Timestamp::now(),
4588 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4589 });
4590 why = Some("rate limited (quota)".to_owned());
4591 }
4592 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4593 why = Some(format!(
4594 "the fixer failed: {}",
4595 e.lines()
4596 .next()
4597 .unwrap_or("")
4598 .chars()
4599 .take(160)
4600 .collect::<String>()
4601 ));
4602 }
4603 }
4604 let agent_id = seat.agent.clone();
4605 self.state.seats.insert(seat.key.clone(), seat);
4606
4607 let mut post = None;
4608 let mut lines = None;
4609 let _coordination = self.coordinate(&wt).await?;
4610 if why.is_none() {
4611 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4612 if let Ok(r) = &rescued {
4613 self.state.note_withheld("graft", &r.withheld);
4614 }
4615 let head = git::rev_parse(&wt, "HEAD").await?;
4616 let dirty = match &rescued {
4620 Ok(r) => git::status_porcelain(&wt)
4621 .await
4622 .map(|st| {
4623 st.lines().any(|l| {
4624 !r.withheld
4625 .iter()
4626 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4627 })
4628 })
4629 .unwrap_or(true),
4630 Err(_) => true,
4631 };
4632 if dirty {
4633 why = Some("the fixer left changes that could not be committed".to_owned());
4634 } else if head == pre {
4635 why = Some("the fixer changed nothing".to_owned());
4636 } else {
4637 lines = git::changed_lines(&wt, &pre, &head).await;
4638 post = Some(head);
4639 }
4640 }
4641 let (adoption, adoption_error) = match &out {
4642 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4643 _ => (None, None),
4644 };
4645 let requested = graft::requested(&plan);
4646 let outcome = match why {
4647 None => {
4648 self.state.event(
4649 "graft",
4650 format!(
4651 "applied: {}",
4652 lines.map_or_else(
4653 || "changed lines unavailable".to_owned(),
4654 |n| format!("{n} changed lines")
4655 )
4656 ),
4657 );
4658 GraftOutcome::Applied
4659 }
4660 Some(why) => {
4661 crate::recovery::restore_repair(&wt, &pre)
4664 .await
4665 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4666 self.state.event(
4667 "graft",
4668 format!("rolled back: {why}; the winner is untouched"),
4669 );
4670 GraftOutcome::RolledBack(why)
4671 }
4672 };
4673 self.state.graft = Some(GraftRecord {
4674 pre,
4675 post,
4676 lines,
4677 limit: None,
4678 agent: Some(agent_id),
4679 outcome,
4680 requested,
4681 adoption,
4682 adoption_error,
4683 });
4684 self.state.save()?;
4685 Ok(())
4686 }
4687
4688 async fn fold_losers(&mut self) -> Result<()> {
4691 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4692 return Ok(());
4693 };
4694 if !self.state.candidate_preservation.is_empty() {
4695 bail!("candidate preservation incomplete; refusing cleanup");
4696 }
4697 let repo = self.state.repo.clone();
4698 let _coordination = self.coordinate(&repo).await?;
4699 let mut folded = Vec::new();
4700 for i in 0..self.state.candidates.len() {
4701 let c = &self.state.candidates[i];
4702 if c.label == winner || c.folded {
4703 continue;
4704 }
4705 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4706 remove_preserved_worktree(&repo, &wt).await?;
4707 if git::branch_exists(&repo, &branch).await?
4708 && !git::branch_delete(&repo, &branch).await?
4709 {
4710 bail!("candidate branch removal refused");
4711 }
4712 self.state.candidates[i].folded = true;
4713 self.state.save()?;
4714 folded.push(label.to_string());
4715 }
4716 let root = self.state.worktree_root();
4718 for j in 1..=self.roles.judges.len() {
4719 let wt = root.join(format!("judge-{j}"));
4720 if wt.exists() {
4721 remove_preserved_worktree(&repo, &wt).await?;
4722 }
4723 }
4724 if self.state.config.graph.advise {
4727 for k in 1..=self.state.config.graph.advisors {
4728 let wt = root.join(format!("advisor-{k}"));
4729 if wt.exists() {
4730 remove_preserved_worktree(&repo, &wt).await?;
4731 }
4732 }
4733 }
4734 if !folded.is_empty() {
4735 self.state
4736 .event("fold", format!("folded candidates {}", folded.join(", ")));
4737 self.state.save()?;
4738 }
4739 Ok(())
4740 }
4741
4742 async fn hold_base_sync(&mut self) -> Result<bool> {
4745 self.hold_base_sync_as(None).await
4746 }
4747
4748 async fn hold_base_sync_kind(
4749 &mut self,
4750 kind: crate::base_sync_recovery::StopKind,
4751 ) -> Result<()> {
4752 self.hold_base_sync_as(Some(kind)).await?;
4753 Ok(())
4754 }
4755
4756 async fn hold_base_sync_as(
4757 &mut self,
4758 kind: Option<crate::base_sync_recovery::StopKind>,
4759 ) -> Result<bool> {
4760 if !self.state.base_sync_blocked() {
4761 self.state.base_sync_recovery.stop = None;
4762 return Ok(false);
4763 }
4764 let head = match self.state.winner() {
4765 Some(w) => git::rev_parse(&w.worktree, "HEAD")
4766 .await
4767 .unwrap_or_default(),
4768 None => String::new(),
4769 };
4770 self.state.record_base_sync_stop(&head);
4771 if let Some(kind) = kind {
4772 self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4773 }
4774 self.state.save()?;
4775 Ok(true)
4776 }
4777
4778 async fn sync_to_base(&mut self) -> Result<()> {
4816 if self.state.status == RunStatus::AlreadyInBase {
4817 return Ok(());
4818 }
4819 self.hold_base_sync().await?;
4820 let conflicted = self
4821 .state
4822 .base_sync
4823 .as_ref()
4824 .is_some_and(|s| s.conflict.is_some());
4825 let Some(winner) = self.state.winner().cloned() else {
4826 return Ok(());
4827 };
4828
4829 let repo = self.state.repo.clone();
4830 let remote = self.state.config.merge.remote.clone();
4831 let base_branch = self.state.base_branch.clone();
4832 let tracking = format!("{remote}/{base_branch}");
4833
4834 let fetched_base =
4835 matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4836 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4841 return Ok(());
4842 };
4843
4844 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4845 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4846 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4847
4848 if (behind > 0 || conflicted || head == tip)
4857 && self
4858 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4859 .await?
4860 {
4861 return Ok(());
4862 }
4863 if conflicted && self.state.base_recovery_pending() {
4864 let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4865 let owner =
4866 crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4867 if !fetched_base
4868 || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4869 || !git::is_clean(&winner.worktree).await?
4870 || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4871 {
4872 return Ok(());
4873 }
4874 }
4875 let recovery = conflicted && self.state.consume_base_recovery(&head);
4876 if conflicted {
4877 self.state.save()?;
4879 if !recovery {
4880 return Ok(());
4881 }
4882 }
4883
4884 if behind == 0 {
4885 if let Some(from) = self
4894 .state
4895 .rebase_fixes
4896 .iter()
4897 .rev()
4898 .find_map(|r| r.from.clone())
4899 && from != head
4900 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4901 .await
4902 .is_ok_and(|o| o.ok())
4903 {
4904 git::sync_to_head(&winner.worktree).await?;
4905 }
4906 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4912 if let Some(why) = &conflict {
4913 self.state.status = RunStatus::Blocked;
4914 self.state.event("land", why.clone());
4915 }
4916 self.state.base_sync = Some(BaseSync {
4917 tip,
4918 behind: 0,
4919 attempts: attempts + usize::from(recovery),
4920 conflict,
4921 already_in: None,
4922 });
4923 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4924 .await?;
4925 self.state.save()?;
4926 return Ok(());
4927 }
4928
4929 if attempts >= BASE_SYNC_ROUNDS && !recovery {
4930 let why = format!(
4931 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4932 rebase(s); rebasing again would only race it",
4933 winner.branch
4934 );
4935 self.state.status = RunStatus::Blocked;
4936 self.state.base_sync = Some(BaseSync {
4937 tip,
4938 behind,
4939 attempts,
4940 conflict: Some(why.clone()),
4941 already_in: None,
4942 });
4943 self.state.event("land", why);
4944 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4945 .await?;
4946 self.state.save()?;
4947 return Ok(());
4948 }
4949
4950 self.state.event(
4951 "land",
4952 format!(
4953 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4954 winner.branch
4955 ),
4956 );
4957 self.state.save()?;
4958
4959 let branch_tracking = format!("{remote}/{}", winner.branch);
4965 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4966 let remote_tip = if self.state.task_pr_branch().is_none()
4967 && matches!(&fetched_branch, Ok(o) if o.ok())
4968 {
4969 git::rev_parse(&repo, &branch_tracking).await.ok()
4970 } else {
4971 None
4972 };
4973 if let Some(theirs) = &remote_tip
4977 && !git::is_ancestor(&repo, theirs, &head).await
4978 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4979 .await
4980 .is_ok_and(|missing| missing.is_empty())
4981 {
4982 let why = format!(
4983 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4984 them",
4985 short(theirs),
4986 winner.branch
4987 );
4988 self.state.status = RunStatus::Blocked;
4989 self.state.base_sync = Some(BaseSync {
4990 tip,
4991 behind,
4992 attempts: attempts + usize::from(recovery),
4993 conflict: Some(why.clone()),
4994 already_in: None,
4995 });
4996 self.state.event("land", why);
4997 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4998 .await?;
4999 self.state.save()?;
5000 return Ok(());
5001 }
5002
5003 let scratch = self.state.dir().join("base-sync");
5004 let rebased =
5005 match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5006 .await
5007 {
5008 Ok(crate::rebase::Rebased::Applied) => Ok(None),
5009 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5010 Err(e) => Err(e),
5011 };
5012 let attempts = attempts + 1;
5013 let stop_kind = match &rebased {
5014 Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5015 Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5016 Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5017 };
5018 match rebased {
5019 Ok(None) => {
5020 git::sync_to_head(&winner.worktree).await?;
5024 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5025 let mut conflict = None;
5026 if let Some(pinned) = &remote_tip {
5027 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5028 match pushed {
5029 Ok(o) if o.ok() => self.state.event(
5030 "land",
5031 format!("pushed rebased {} to {remote}", winner.branch),
5032 ),
5033 Ok(o) => {
5034 conflict = Some(format!(
5035 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
5036 winner.branch,
5037 short(pinned),
5038 o.stderr.chars().take(600).collect::<String>()
5039 ));
5040 }
5041 Err(e) => {
5042 conflict = Some(format!(
5043 "rebased {} locally but could not push it: {e:#}",
5044 winner.branch
5045 ));
5046 }
5047 }
5048 }
5049 if let Some(why) = &conflict {
5050 self.state.status = RunStatus::Blocked;
5051 self.state.event("land", why.clone());
5052 }
5053 self.state.base_sync = Some(BaseSync {
5054 tip: tip.clone(),
5055 behind: 0,
5056 attempts,
5057 conflict,
5058 already_in: None,
5059 });
5060 self.state
5061 .event("land", format!("rebased {} onto {tracking}", winner.branch));
5062 }
5063 Ok(Some(conflict)) => {
5064 let why = format!(
5065 "{} conflicts with {tracking} and did not rebase: {}",
5066 winner.branch,
5067 conflict.chars().take(600).collect::<String>()
5068 );
5069 self.state.status = RunStatus::Blocked;
5070 self.state.base_sync = Some(BaseSync {
5071 tip,
5072 behind,
5073 attempts,
5074 conflict: Some(why.clone()),
5075 already_in: None,
5076 });
5077 self.state.event("land", why);
5078 }
5079 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5083 self.state.save()?;
5084 return Ok(());
5085 }
5086 Err(e) => {
5087 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5088 self.state.status = RunStatus::Blocked;
5089 self.state.base_sync = Some(BaseSync {
5090 tip,
5091 behind,
5092 attempts,
5093 conflict: Some(why.clone()),
5094 already_in: None,
5095 });
5096 self.state.event("land", why);
5097 }
5098 }
5099 if !self.state.base_sync_blocked() {
5100 self.state.base_sync_recovery.stop = None;
5101 }
5102 self.hold_base_sync_kind(stop_kind).await?;
5103 self.state.save()?;
5104 Ok(())
5105 }
5106
5107 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5113 if self.state.task_pr_branch().is_some() {
5116 return None;
5117 }
5118 let repo = self.state.repo.clone();
5119 let remote = self.state.config.merge.remote.clone();
5120 let fetched = git::fetch(&repo, &remote, branch).await;
5121 if !matches!(&fetched, Ok(o) if o.ok()) {
5122 return None;
5123 }
5124 let branch_tracking = format!("{remote}/{branch}");
5125 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5126 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5127 return None;
5128 }
5129 if !crate::reconcile::origin_missing(&repo, head, &theirs)
5130 .await
5131 .is_ok_and(|missing| missing.is_empty())
5132 {
5133 return Some(format!(
5134 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5135 them",
5136 short(&theirs)
5137 ));
5138 }
5139 match git::push_pinned(&repo, &remote, branch, &theirs).await {
5140 Ok(o) if o.ok() => {
5141 self.state
5142 .event("land", format!("pushed rebased {branch} to {remote}"));
5143 None
5144 }
5145 Ok(o) => Some(format!(
5146 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5147 someone may have pushed): {}",
5148 short(&theirs),
5149 o.stderr.chars().take(600).collect::<String>()
5150 )),
5151 Err(e) => Some(format!(
5152 "{branch} is rebased locally but could not be pushed: {e:#}"
5153 )),
5154 }
5155 }
5156
5157 async fn settle_already_in(
5174 &mut self,
5175 branch: &str,
5176 tip: &str,
5177 head: &str,
5178 attempts: usize,
5179 behind: usize,
5180 ) -> Result<bool> {
5181 let repo = self.state.repo.clone();
5182 let remote = self.state.config.merge.remote.clone();
5183 let start = self.state.base_commit.clone();
5184 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5185 Ok(Some(e)) => e,
5186 Ok(None) => return Ok(false),
5187 Err(e) => {
5188 tracing::warn!("already-in-base check for {branch}: {e:#}");
5189 return Ok(false);
5190 }
5191 };
5192 let mut verified = vec![head.to_owned()];
5193 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5194 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5195 if matches!(&fetched, Ok(o) if o.ok())
5196 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5197 && theirs != head
5198 {
5199 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5200 Ok(Some(_)) => verified.push(theirs),
5201 _ => return Ok(false),
5202 }
5203 }
5204 let base_branch = self.state.base_branch.clone();
5205 let message = format!(
5206 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5207 land",
5208 evidence.names(),
5209 evidence.proof.as_str()
5210 );
5211 let closed =
5212 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5213 .await;
5214 match closed {
5215 Ok(Ok(url)) => self
5216 .state
5217 .event("land", format!("closed {url}: superseded on {base_branch}")),
5218 Ok(Err(why)) => self
5219 .state
5220 .event("land", format!("did not close a pull request: {why}")),
5221 Err(e) => {
5222 let why = format!(
5223 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5224 not be closed ({e:#}); resume to retry"
5225 );
5226 self.state.status = RunStatus::Blocked;
5227 self.state.base_sync = Some(BaseSync {
5228 tip: tip.to_owned(),
5229 behind,
5230 attempts,
5231 conflict: Some(why.clone()),
5232 already_in: None,
5233 });
5234 self.state.event("land", why);
5235 self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5236 .await?;
5237 self.state.save()?;
5238 return Ok(true);
5239 }
5240 }
5241 self.state.status = RunStatus::AlreadyInBase;
5242 self.state.base_sync_recovery.stop = None;
5243 self.state.base_sync = Some(BaseSync {
5244 tip: tip.to_owned(),
5245 behind,
5246 attempts,
5247 conflict: None,
5248 already_in: Some(evidence),
5249 });
5250 self.state.event("land", message);
5251 self.state.save()?;
5252 self.settle_questions();
5253 Ok(true)
5254 }
5255
5256 fn landing_base(&self) -> String {
5266 self.state
5267 .base_sync
5268 .as_ref()
5269 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5270 }
5271
5272 pub async fn fix_selected(
5305 &mut self,
5306 ids: &[String],
5307 reason: &str,
5308 allow_stale: bool,
5309 ) -> Result<()> {
5310 let reason = reason.trim();
5311 if reason.is_empty() {
5312 bail!("a fix request needs a reason — that is the operator's own record of why");
5313 }
5314 if ids.is_empty() {
5315 bail!("no finding id given");
5316 }
5317 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5318 bail!(
5319 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5320 has already concluded — can be given a targeted fix. A run still \
5321 in progress should simply be resumed; a `merged` run's branch has \
5322 already landed, so its answer is a fresh `magi review <branch>`, \
5323 not reopening this run's own record",
5324 self.state.id,
5325 self.state.status.as_str()
5326 );
5327 }
5328 let Some(winner) = self.state.winner().cloned() else {
5329 bail!("run {} has no winning candidate to fix", self.state.id);
5330 };
5331 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5332 bail!(
5333 "branch `{}` no longer exists; this run cannot be extended",
5334 winner.branch
5335 );
5336 }
5337 let home = crate::run::home();
5338 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5339 bail!(
5340 "run {} is currently being worked on by another magi process",
5341 self.state.id
5342 );
5343 }
5344 let _claim = FixClaim::acquire(&self.state.dir())?;
5350
5351 let mut seen = BTreeSet::new();
5355 let mut findings = Vec::new();
5356 let mut missing = Vec::new();
5357 for id in ids {
5358 if !seen.insert(id.clone()) {
5359 continue;
5360 }
5361 match self.state.finding(id) {
5362 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5363 id: f.id.clone(),
5364 severity: f.severity,
5365 reviewer_vote: rec.vote,
5366 round: round.round,
5367 round_head: round.head.clone(),
5368 reviewer: rec.reviewer,
5369 agent: rec.agent.clone(),
5370 file: f.file.clone(),
5371 line: f.line,
5372 title: f.title.clone(),
5373 detail: f.detail.clone(),
5374 outcome: OperatorFixOutcome::Pending,
5375 }),
5376 None => missing.push(id.clone()),
5377 }
5378 }
5379 if !missing.is_empty() {
5380 bail!(
5381 "unknown finding id(s): {}; nothing was changed",
5382 missing.join(", ")
5383 );
5384 }
5385
5386 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5387 let stale_details: Vec<(String, String)> = findings
5388 .iter()
5389 .filter(|f| f.round_head != head_at_request)
5390 .map(|f| (f.id.clone(), f.round_head.clone()))
5391 .collect();
5392 let stale = !stale_details.is_empty();
5393 if stale && !allow_stale {
5394 bail!(
5395 "the branch has moved since some finding(s) were raised — {} — now \
5396 at {}; pass --allow-stale to fix anyway, or re-run review first",
5397 stale_details
5398 .iter()
5399 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5400 .collect::<Vec<_>>()
5401 .join(", "),
5402 short(&head_at_request)
5403 );
5404 }
5405
5406 let request = OperatorFixRequest {
5407 requested_at: Timestamp::now(),
5408 reason: reason.to_owned(),
5409 findings,
5410 head_at_request: head_at_request.clone(),
5411 allow_stale,
5412 stale,
5413 fix: None,
5414 result_head: None,
5415 follow_up_review_run: None,
5416 };
5417 self.state.event(
5418 "fix",
5419 format!(
5420 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5421 request.findings.len(),
5422 request
5423 .findings
5424 .iter()
5425 .map(|f| f.id.as_str())
5426 .collect::<Vec<_>>()
5427 .join(", "),
5428 ),
5429 );
5430 self.state.operator_fixes.push(request);
5437 self.state.save()?;
5438 let request_index = self.state.operator_fixes.len() - 1;
5439
5440 let repo = self.state.repo.clone();
5449 let coordination = self.coordinate(&repo).await?;
5450 if winner.worktree.exists() {
5451 let dirty = git::git(
5454 &winner.worktree,
5455 &["status", "--porcelain", "--untracked-files=all"],
5456 )
5457 .await?;
5458 let only_withheld = dirty.lines().all(|l| {
5459 l.strip_prefix("?? ")
5460 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5461 });
5462 if !only_withheld {
5463 bail!(
5464 "`{}` has uncommitted changes; refusing to touch it — commit or \
5465 discard them first",
5466 winner.worktree.display()
5467 );
5468 }
5469 git::worktree_remove(&self.state.repo, &winner.worktree)
5470 .await
5471 .ok();
5472 }
5473 let fix_worktree = self.state.worktree_root().join("operator-fix");
5474 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5475 git::git(
5476 &self.state.repo,
5477 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5478 )
5479 .await
5480 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5481 if !git::is_clean(&fix_worktree).await? {
5482 git::worktree_remove(&self.state.repo, &fix_worktree)
5483 .await
5484 .ok();
5485 bail!(
5486 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5487 winner.branch
5488 );
5489 }
5490 let repo_for_setup = self.state.repo.clone();
5491 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5492 Ok(()) => {
5493 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5494 }
5495 Err(e) => Err(e),
5496 };
5497 if let Err(e) = setup {
5498 git::worktree_remove(&self.state.repo, &fix_worktree)
5499 .await
5500 .ok();
5501 return Err(e);
5502 }
5503
5504 let run_id = self.state.id.clone();
5505 let prompts = self.state.config.prompts.clone();
5506 let language = self.state.config.graph.language.clone();
5507 let sessions = self.state.config.graph.sessions;
5508 let artifacts = agent::artifacts_dir(&self.state.dir());
5509 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5510 .findings
5511 .iter()
5512 .map(|f| Finding {
5513 id: f.id.clone(),
5514 severity: f.severity,
5515 file: f.file.clone(),
5516 line: f.line,
5517 title: f.title.clone(),
5518 detail: f.detail.clone(),
5519 })
5520 .collect();
5521 let fix_prompt = prompt::operator_fix_in(
5522 &self.state.instruction,
5523 &finding_list,
5524 reason,
5525 &stale_details,
5526 &head_at_request,
5527 &language,
5528 &self.state.config.graph.github_language,
5529 );
5530 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5531 drop(coordination);
5532 let (job, seat, out) = self
5533 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5534 prompt: fix_prompt.clone(),
5535 spec,
5536 seat,
5537 cwd: fix_worktree.clone(),
5538 timeout,
5539 allow_write: true,
5540 sessions,
5541 artifacts: artifacts.clone(),
5542 stem: "operator-fix".to_owned(),
5543 handover: None,
5544 })
5545 .await;
5546 let agent_id = seat.agent.clone();
5547
5548 let mut fix = FixRecord {
5549 agent: agent_id,
5550 addressed: Vec::new(),
5551 rejected: Vec::new(),
5552 notes: String::new(),
5553 committed: false,
5554 failed: None,
5555 duration_ms: 0,
5556 continuation: None,
5557 };
5558 let mut final_seat = seat.clone();
5559 match out {
5560 AgentOutcome::Ok(o) => {
5561 fix.duration_ms = o.duration_ms;
5562 let parsed = verdict::extract_json::<FixReport>(&o.text);
5563 let incomplete_reason = match &parsed {
5564 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5565 "the reply parsed, but it reported a command whose own CLI \
5566 never confirmed an exit status"
5567 .to_owned(),
5568 ),
5569 Ok(_) => None,
5570 Err(e) => Some(e.to_string()),
5571 };
5572 match incomplete_reason {
5573 None => {
5574 let report = parsed.expect("checked Ok above");
5575 fix.addressed = report.addressed;
5576 fix.rejected = report.rejected;
5577 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5578 }
5579 Some(reason) => {
5580 let (resumed_seat, resolved, failure, cont) = self
5581 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5582 .await;
5583 fix.duration_ms += cont.cumulative_wait_ms;
5584 fix.continuation = Some(cont);
5585 final_seat = resumed_seat;
5586 match resolved {
5587 Some(report) => {
5588 fix.addressed = report.addressed;
5589 fix.rejected = report.rejected;
5590 fix.notes =
5591 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5592 }
5593 None => fix.failed = failure,
5594 }
5595 }
5596 }
5597 }
5598 AgentOutcome::Dropped(o) => {
5599 fix.duration_ms = o.duration_ms;
5600 let why = o
5601 .dropped
5602 .as_ref()
5603 .map(|d| d.why.as_str())
5604 .unwrap_or("the CLI ended the stream without delivering its answer");
5605 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5606 }
5607 AgentOutcome::Quota(o) => {
5608 self.state.quota.push(QuotaLoss {
5609 seat: final_seat.key.clone(),
5610 node: "fix".to_owned(),
5611 at: Timestamp::now(),
5612 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5613 });
5614 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5615 }
5616 AgentOutcome::Failed(e) => fix.failed = Some(e),
5617 AgentOutcome::ResourceBlocked(e) => {
5618 fix.failed = Some(format!("the fixer was not started: {e}"));
5619 }
5620 }
5621 if fix.continuation.is_none() {
5622 fix.continuation = Some(ContinuationRecord::not_needed());
5623 }
5624 self.state.seats.insert(final_seat.key.clone(), final_seat);
5625
5626 let rescue_message = format!(
5627 "magi: operator-selected fix ({}) (uncommitted work)",
5628 self.state.operator_fixes[request_index]
5629 .findings
5630 .iter()
5631 .map(|f| f.id.as_str())
5632 .collect::<Vec<_>>()
5633 .join(", ")
5634 );
5635 let repo = self.state.repo.clone();
5636 let coordination = self.coordinate(&repo).await?;
5637 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5638 self.state.note_withheld("fix", &r.withheld);
5639 }
5640 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5641 fix.committed = after != head_at_request;
5642 git::worktree_remove(&self.state.repo, &fix_worktree)
5643 .await
5644 .ok();
5645 drop(coordination);
5646
5647 self.state.event(
5648 "fix",
5649 match &fix.failed {
5650 Some(reason) => format!(
5651 "operator fix: adoption report was lost ({reason}); {}",
5652 if fix.committed {
5653 "committed"
5654 } else {
5655 "NO new commit"
5656 }
5657 ),
5658 None => format!(
5659 "operator fix: {} addressed, {} rejected, {}",
5660 fix.addressed.len(),
5661 fix.rejected.len(),
5662 if fix.committed {
5663 "committed"
5664 } else {
5665 "NO new commit"
5666 }
5667 ),
5668 },
5669 );
5670
5671 for f in &mut self.state.operator_fixes[request_index].findings {
5678 f.outcome = if fix.failed.is_some() {
5679 OperatorFixOutcome::Unreported
5680 } else if fix.addressed.contains(&f.id) {
5681 OperatorFixOutcome::Addressed
5682 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5683 OperatorFixOutcome::Rejected { why: r.why.clone() }
5684 } else {
5685 OperatorFixOutcome::Unreported
5686 };
5687 }
5688
5689 let committed = fix.committed;
5690 if committed {
5691 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5692 }
5693 self.state.operator_fixes[request_index].fix = Some(fix);
5694 self.state.save()?;
5697
5698 if committed {
5699 self.state.event(
5700 "fix",
5701 format!(
5702 "operator fix committed {}; opening a follow-up review-only run",
5703 short(&after)
5704 ),
5705 );
5706 let origin =
5709 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5710 match Self::review(
5711 &self.state.repo,
5712 &winner.branch,
5713 self.state.config.clone(),
5714 origin,
5715 )
5716 .await
5717 {
5718 Ok(mut follow_up) => {
5719 follow_up.state.event(
5720 "start",
5721 format!(
5722 "requested by an operator fix on run {} for finding(s) {}",
5723 self.state.id,
5724 self.state.operator_fixes[request_index]
5725 .findings
5726 .iter()
5727 .map(|f| f.id.as_str())
5728 .collect::<Vec<_>>()
5729 .join(", "),
5730 ),
5731 );
5732 follow_up.state.save()?;
5733 let follow_up_id = follow_up.state.id.clone();
5734 let adopted = match crate::direct::adopt(
5737 &follow_up.state,
5738 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5739 ) {
5740 Ok(a) => a,
5741 Err(e) => {
5742 self.state.event(
5745 "fix",
5746 format!(
5747 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5748 ),
5749 );
5750 self.state.save()?;
5751 return Ok(());
5752 }
5753 };
5754 let executed = follow_up.execute().await;
5755 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5756 if let Some(a) = adopted {
5757 a.finish(&follow_up.state, executed);
5758 }
5759 if let Some(e) = failure {
5760 self.state.event(
5761 "fix",
5762 format!(
5763 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5764 ),
5765 );
5766 }
5767 self.state.operator_fixes[request_index].follow_up_review_run =
5768 Some(follow_up_id);
5769 }
5770 Err(e) => {
5771 self.state.event(
5772 "fix",
5773 format!("committed the fix but could not open a follow-up review: {e:#}"),
5774 );
5775 }
5776 }
5777 self.state.save()?;
5778 }
5779
5780 Ok(())
5781 }
5782
5783 async fn ask_fixer(
5803 &mut self,
5804 winner: &Candidate,
5805 node: &'static str,
5806 round: Option<usize>,
5807 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5808 ) -> (SeatJob, SeatState, AgentOutcome) {
5809 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5810 self.ask_fixer_among(attempts, node, round, None, build)
5811 .await
5812 }
5813
5814 async fn ask_fixer_among(
5819 &mut self,
5820 attempts: Vec<(AgentSpec, String)>,
5821 node: &'static str,
5822 round: Option<usize>,
5823 restore: Option<(&Path, &str)>,
5824 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5825 ) -> (SeatJob, SeatState, AgentOutcome) {
5826 let run_id = self.state.id.clone();
5827 let prompts = self.state.config.prompts.clone();
5828 let cache = self.state.config.cache_dir();
5829 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5830 let mut last = None;
5831 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5832 let seat = self.seat(&key, &spec.id);
5833 let mut job = build(spec.clone(), seat);
5834 if i > 0 {
5835 job.stem = format!("{}-{}", job.stem, spec.id);
5836 }
5837 let ctx = WaveCtx {
5838 carry_seats: false,
5839 pauses: Some((&self.pause, &self.interrupt)),
5840 run: &run_id,
5841 node,
5842 prompts: &prompts,
5843 cache: cache.as_deref(),
5844 round,
5845 };
5846 let (seat, out) =
5847 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5848 let advances = match &out {
5851 AgentOutcome::Ok(o) => agent::output_advances(o),
5852 AgentOutcome::ResourceBlocked(_) => false,
5853 _ => true,
5854 };
5855 if let Some(next) = ids.get(i + 1)
5856 && advances
5857 {
5858 self.state.seats.insert(seat.key.clone(), seat.clone());
5859 let class =
5860 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5861 let reason = match &out {
5862 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5863 other => fail_reason(other),
5864 };
5865 if let Some((wt, rev)) = restore
5869 && let Err(e) = self.graft_restore(wt, rev).await
5870 {
5871 let why =
5872 format!("could not restore the worktree before the next fixer: {e:#}");
5873 tracing::warn!("{why}");
5874 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5875 AgentOutcome::ResourceBlocked(why)
5876 } else {
5877 AgentOutcome::Failed(why)
5878 };
5879 last = Some((job, seat, out));
5880 break;
5881 }
5882 record_handover(
5883 &mut self.state,
5884 node,
5885 &seat.key,
5886 &spec.id,
5887 next,
5888 &class,
5889 &reason,
5890 );
5891 continue;
5892 }
5893 last = Some((job, seat, out));
5894 break;
5895 }
5896 last.expect("the fixer chain always has an entry")
5897 }
5898
5899 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5910 use crate::recovery::{Mode, Repair};
5911 if !self.state.reviews.is_empty() {
5912 return Ok(());
5913 }
5914 let Some(mut rec) = self.state.recovery.clone() else {
5915 return Ok(());
5916 };
5917 if rec.mode != Mode::VerifyRepair {
5918 return Ok(());
5919 }
5920 let wt = winner.worktree.clone();
5921 let mut head = git::rev_parse(&wt, "HEAD").await?;
5922 match rec.repair.clone() {
5923 Repair::Done { .. } => return Ok(()),
5924 Repair::WaitingForRestore { before } => {
5925 self.graft_restore(&wt, &before).await?;
5926 head = before;
5927 rec.repair = Repair::NotStarted;
5928 self.state.recovery = Some(rec.clone());
5929 self.state.save()?;
5930 }
5931 Repair::Started { before } => {
5932 if let Err(e) = self.graft_restore(&wt, &before).await {
5936 if e.is::<crate::repo_lock::ResourceBlocked>() {
5937 return Err(e);
5938 }
5939 return self
5940 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5941 .await;
5942 }
5943 rec.repair = Repair::Done {
5944 before: before.clone(),
5945 after: before,
5946 committed: false,
5947 failed: Some("interrupted; rolled back".to_owned()),
5948 };
5949 self.state
5950 .event("recovery", "the repair was interrupted and rolled back");
5951 self.state.recovery = Some(rec);
5952 self.state.save()?;
5953 return Ok(());
5954 }
5955 Repair::NotStarted => {}
5956 }
5957 if crate::recovery::recheck(&mut rec, &head) {
5958 self.state.event("recovery", rec.summary());
5959 self.state.recovery = Some(rec);
5960 self.state.save()?;
5961 return Ok(());
5962 }
5963 rec.repair = Repair::Started {
5964 before: head.clone(),
5965 };
5966 self.state.recovery = Some(rec.clone());
5967 self.state.save()?;
5968
5969 let language = self.state.config.graph.language.clone();
5970 let sessions = self.state.config.graph.sessions;
5971 let artifacts = agent::artifacts_dir(&self.state.dir());
5972 let fix_prompt = prompt::recovery_fix(
5973 &self.state.instruction,
5974 &rec,
5975 self.state.config.graph.review_rounds,
5976 &language,
5977 );
5978 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5979 self.state.event(
5980 "recovery",
5981 format!(
5982 "repair: the fixer starts from the failed verification on {}",
5983 short(&head)
5984 ),
5985 );
5986 let (_job, seat, out) = self
5987 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5988 prompt: fix_prompt.clone(),
5989 spec,
5990 seat,
5991 cwd: wt.clone(),
5992 timeout,
5993 allow_write: true,
5994 sessions,
5995 artifacts: artifacts.clone(),
5996 stem: "fix-recovery".to_owned(),
5997 handover: None,
5998 })
5999 .await;
6000 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6001 rec.repair = Repair::WaitingForRestore { before: head };
6002 self.state.recovery = Some(rec);
6003 return self.stop_for_resource();
6004 }
6005 let failed = match &out {
6006 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6007 AgentOutcome::Quota(o) => {
6008 self.state.quota.push(QuotaLoss {
6009 seat: seat.key.clone(),
6010 node: "fix".to_owned(),
6011 at: Timestamp::now(),
6012 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6013 });
6014 Some("rate limited (quota); the fixer could not run".to_owned())
6015 }
6016 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6017 AgentOutcome::Failed(e) => Some(e.clone()),
6018 };
6019 self.state.seats.insert(seat.key.clone(), seat);
6020 let coordination = self.coordinate(&wt).await?;
6021 let failed = match failed {
6022 Some(why) => {
6024 if let Err(e) = git::restore_to(&wt, &head).await {
6025 return self
6026 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6027 .await;
6028 }
6029 Some(why)
6030 }
6031 None => {
6032 if let Ok(r) = git::rescue_commit(
6033 &wt,
6034 "magi: recovery repair of the failed verification (uncommitted work)",
6035 )
6036 .await
6037 {
6038 self.state.note_withheld("fix", &r.withheld);
6039 }
6040 None
6041 }
6042 };
6043 let after = git::rev_parse(&wt, "HEAD").await?;
6044 drop(coordination);
6045 self.state.event(
6046 "recovery",
6047 match &failed {
6048 Some(why) => format!("repair: nothing kept ({why})"),
6049 None if after != head => {
6050 format!("repair: committed {}", short(&after))
6051 }
6052 None => "repair: the fixer made no commit".to_owned(),
6053 },
6054 );
6055 rec.repair = Repair::Done {
6056 before: head.clone(),
6057 committed: after != head,
6058 after,
6059 failed,
6060 };
6061 self.state.recovery = Some(rec);
6062 self.state.save()?;
6063 Ok(())
6064 }
6065
6066 async fn stop_recovery(
6069 &mut self,
6070 mut rec: crate::recovery::RecoveryRecord,
6071 head: &str,
6072 why: String,
6073 ) -> Result<()> {
6074 rec.repair = crate::recovery::Repair::Started {
6076 before: head.to_owned(),
6077 };
6078 self.state.recovery = Some(rec);
6079 self.state
6080 .event("recovery", format!("repair could not be undone: {why}"));
6081 self.state.status = RunStatus::Blocked;
6082 self.state.save()?;
6083 bail!("recovery repair could not be rolled back: {why}")
6084 }
6085
6086 async fn review_loop(&mut self) -> Result<()> {
6087 if self.hold_base_sync().await? {
6088 return Ok(());
6089 }
6090 loop {
6091 if self
6092 .state
6093 .base_sync
6094 .as_ref()
6095 .is_some_and(|s| s.conflict.is_some())
6096 {
6097 return Ok(());
6098 }
6099 if self.catchup_repairable() {
6100 let record = self.state.reviews.last().expect("repairable round").clone();
6101 let winner = self
6102 .state
6103 .winner()
6104 .cloned()
6105 .context("catch-up repair has no winner")?;
6106 if !self.fix_review_round(&winner, record).await? {
6107 return Ok(());
6108 }
6109 }
6110 self.review_loop_pass().await?;
6111 if !self.catchup_repairable() {
6112 return Ok(());
6113 }
6114 }
6115 }
6116
6117 fn catchup_repairable(&self) -> bool {
6120 self.state.reviews.len() < self.state.config.graph.review_rounds
6121 && self.state.reviews.last().is_some_and(|r| {
6122 !r.clean
6123 && r.blocking == 0
6124 && r.fix.is_none()
6125 && r.e2e_status() == E2eStatus::Failed
6126 })
6127 }
6128
6129 async fn review_loop_pass(&mut self) -> Result<()> {
6130 if self
6135 .state
6136 .base_sync
6137 .as_ref()
6138 .is_some_and(|s| s.conflict.is_some())
6139 {
6140 return Ok(());
6141 }
6142 let run_id = self.state.id.clone();
6147 let prompts = self.state.config.prompts.clone();
6148 let Some(winner) = self.state.winner().cloned() else {
6149 return Ok(());
6150 };
6151 let max_rounds = self.state.config.graph.review_rounds;
6152 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6162 if status == RunStatus::Gating {
6165 self.record_contested_handoff();
6166 }
6167 self.state.status = status;
6168 self.state.save()?;
6169 return Ok(());
6170 }
6171 self.state.status = RunStatus::Reviewing;
6172 if self.state.reviews.last().is_some_and(|r| {
6182 r.e2e_status() == E2eStatus::ResourceBlocked
6183 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6184 }) {
6185 let shell = self.state.config.shell();
6186 return self
6187 .stop_unresolved_review(
6188 "the last round's own verification never resolved",
6189 &shell,
6190 &winner.worktree,
6191 )
6192 .await;
6193 }
6194
6195 self.recovery_repair(&winner).await?;
6196 if self.resource_halt()? {
6197 return Ok(());
6198 }
6199
6200 let repo = self.state.repo.clone();
6201 let root = self.state.worktree_root();
6202 let language = self.state.config.graph.language.clone();
6203 let sessions = self.state.config.graph.sessions;
6204 let artifacts = self.merge_repair_artifacts();
6205 let base = self.landing_base();
6206 let base_short = short(&base);
6207 let reviewers = self.roles.reviewers.clone();
6208 let shell = self.state.config.shell();
6209
6210 for round in (self.state.reviews.len() + 1)..=max_rounds {
6211 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6212 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6213 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6214 let prev_verification = self
6223 .state
6224 .reviews
6225 .last()
6226 .and_then(|r| r.verification_summary(&head));
6227 let prior_section = match (&self.state.recovery, round) {
6230 (Some(rec), 1) if self.state.reviews.is_empty() => {
6231 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6232 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6233 Some(crate::recovery::head_of(
6234 &git::diff(&winner.worktree, reviewed, "HEAD")
6235 .await
6236 .unwrap_or_default(),
6237 crate::recovery::SINCE_DIFF_MAX,
6238 ))
6239 }
6240 _ => None,
6241 };
6242 Some(prompt::recovery_review(rec, since.as_deref()))
6243 }
6244 _ => None,
6245 };
6246
6247 let mut jobs = Vec::new();
6251 let coordination = self.coordinate(&repo).await?;
6252 for (r, spec) in reviewers.iter().cloned().enumerate() {
6253 let wt = root.join(format!("review-{}", r + 1));
6254 if wt.exists() {
6257 git::reset_detached(&wt, &head).await?;
6258 } else {
6259 git::worktree_add_detached(&repo, &wt, &head).await?;
6260 }
6261 self.setup_seat_worktree(&repo, &wt, false).await?;
6262 let seat_key = format!("review-{}", r + 1);
6263 let spec = pick_start_spec(
6266 &self.roles.reviewer_roster,
6267 spec,
6268 self.state.seat_history.get(&seat_key),
6269 );
6270 let seat = self.seat(&seat_key, &spec.id);
6271 jobs.push(SeatJob {
6272 prompt: prompt::review(&prompt::ReviewCtx {
6273 instruction: &self.state.instruction,
6274 branch: &winner.branch,
6275 base_short: &base_short,
6276 stat: &stat,
6277 patch: &patch,
6278 verification: prev_verification.as_ref(),
6279 prior: prior_section.as_deref(),
6280 reviewers: reviewers.len(),
6281 round,
6282 rounds: max_rounds,
6283 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6286 lens: Lens::for_seat(r),
6287 language: &language,
6288 github_language: &self.state.config.graph.github_language,
6289 }),
6290 spec,
6291 seat,
6292 cwd: wt,
6293 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6294 allow_write: false,
6295 sessions,
6296 artifacts: artifacts.clone(),
6297 stem: format!("review-{round}-{}", r + 1),
6298 handover: None,
6299 });
6300 }
6301
6302 drop(coordination);
6303 self.state.event(
6304 "review",
6305 format!(
6306 "round {round}: {} reviewers on {}",
6307 jobs.len(),
6308 short(&head)
6309 ),
6310 );
6311 let mut quota_losses = Vec::new();
6312 let review_retries = self.state.config.graph.retries;
6313 let review_cache = self.state.config.cache_dir();
6314 let ctx = WaveCtx {
6315 carry_seats: true,
6316 pauses: Some((&self.pause, &self.interrupt)),
6317 run: &run_id,
6318 node: "review",
6319 prompts: &prompts,
6320 cache: review_cache.as_deref(),
6321 round: Some(round),
6322 };
6323 let results = ask_json_wave::<Review>(
6324 jobs,
6325 Arc::clone(&self.sem),
6326 review_retries,
6327 &self.roles.reviewer_roster,
6328 &ctx,
6329 &mut quota_losses,
6330 &mut self.state,
6331 &|_: &Review| Ok(()),
6332 )
6333 .await;
6334 if self.state.resource_block.is_some() {
6335 return self.stop_for_resource();
6336 }
6337 let round_quota_missing = quota_losses.len();
6341 self.state.quota.extend(quota_losses);
6342
6343 let mut records = Vec::new();
6344 let mut all_findings = Vec::new();
6345 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6346 let agent_id = seat.agent.clone();
6347 self.state.seats.insert(seat.key.clone(), seat);
6348 let mut record = ReviewRecord {
6349 reviewer: r + 1,
6350 agent: agent_id,
6351 summary: String::new(),
6352 findings: Vec::new(),
6353 vote: None,
6354 failed: None,
6355 duration_ms: 0,
6356 attempts,
6362 };
6363 match res {
6364 Ok((review, out)) => {
6365 record.summary =
6373 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6374 record.vote = Some(review.vote);
6375 record.duration_ms = out.duration_ms;
6376 for (n, mut f) in review.findings.into_iter().enumerate() {
6377 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6380 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6381 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6382 f.file = f
6388 .file
6389 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6390 all_findings.push(f.clone());
6391 record.findings.push(f);
6392 }
6393 self.state.event(
6394 "review",
6395 format!(
6396 "round {round}: reviewer {} voted {} with {} finding(s)",
6397 r + 1,
6398 review.vote.label(),
6399 record.findings.len()
6400 ),
6401 );
6402 }
6403 Err(e) => {
6404 record.failed = Some(e.to_string());
6405 self.state.event(
6406 "review",
6407 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6408 );
6409 }
6410 }
6411 records.push(record);
6412 }
6413
6414 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6421 let vote_split =
6422 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6423 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6424 if vote_split {
6425 self.state.event(
6426 "review",
6427 format!(
6428 "round {round}: votes split ({}) — one round of reconsideration",
6429 initial_votes
6430 .iter()
6431 .map(|v| v.label())
6432 .collect::<Vec<_>>()
6433 .join(", ")
6434 ),
6435 );
6436 let panel: Vec<ReviewSeatReport<'_>> = records
6439 .iter()
6440 .filter_map(|r| {
6441 r.vote.map(|vote| ReviewSeatReport {
6442 reviewer: r.reviewer,
6443 vote,
6444 summary: &r.summary,
6445 findings: &r.findings,
6446 })
6447 })
6448 .collect();
6449
6450 let mut jobs = Vec::new();
6451 let mut seats_at = Vec::new();
6452 for (r, spec) in reviewers.iter().cloned().enumerate() {
6453 if records[r].vote.is_none() {
6457 continue;
6458 }
6459 let wt = root.join(format!("review-{}", r + 1));
6460 let seat_key = format!("review-{}", r + 1);
6461 let spec = self.occupant(&seat_key, spec);
6462 let seat = self.seat(&seat_key, &spec.id);
6463 let build = |with_patch: bool| {
6471 prompt::review_reconsider(&ReviewReconsiderCtx {
6472 instruction: &self.state.instruction,
6473 reviewer: r + 1,
6474 lens: Lens::for_seat(r),
6475 panel: &panel,
6476 patch: with_patch.then_some(ReviewPatch {
6477 branch: &winner.branch,
6478 base_short: &base_short,
6479 stat: &stat,
6480 patch: &patch,
6481 }),
6482 round,
6483 rounds: max_rounds,
6484 language: &language,
6485 })
6486 };
6487 let full = build(true);
6488 let prompt = if has_context(&spec, &seat, sessions) {
6489 build(false)
6490 } else {
6491 full.clone()
6492 };
6493 jobs.push(SeatJob {
6494 prompt,
6495 spec,
6496 seat,
6497 cwd: wt,
6498 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6499 allow_write: false,
6500 sessions,
6501 artifacts: artifacts.clone(),
6502 stem: format!("review-{round}-reconsider-{}", r + 1),
6503 handover: Some(full),
6504 });
6505 seats_at.push(r);
6506 }
6507
6508 let mut recon_quota_losses = Vec::new();
6509 let recon_cache = self.state.config.cache_dir();
6510 let recon_ctx = WaveCtx {
6511 carry_seats: true,
6512 pauses: Some((&self.pause, &self.interrupt)),
6513 run: &run_id,
6514 node: "review",
6515 prompts: &prompts,
6516 cache: recon_cache.as_deref(),
6517 round: Some(round),
6518 };
6519 let recon_results = ask_json_wave::<ReviewRevote>(
6520 jobs,
6521 Arc::clone(&self.sem),
6522 review_retries,
6523 &self.roles.reviewer_roster,
6524 &recon_ctx,
6525 &mut recon_quota_losses,
6526 &mut self.state,
6527 &|_: &ReviewRevote| Ok(()),
6528 )
6529 .await;
6530 if self.state.resource_block.is_some() {
6531 return self.stop_for_resource();
6532 }
6533 self.state.quota.extend(recon_quota_losses);
6534
6535 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6536 let agent_id = seat.agent.clone();
6537 self.state.seats.insert(seat.key.clone(), seat);
6538 let mut rec = ReviewRevoteRecord {
6539 reviewer: r + 1,
6540 agent: agent_id,
6541 vote: None,
6542 reason: String::new(),
6543 failed: None,
6544 };
6545 match res {
6546 Ok((rv, _)) => {
6547 rec.vote = Some(rv.vote);
6548 rec.reason =
6549 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6550 self.state.event(
6551 "review",
6552 format!(
6553 "round {round}: reviewer {} revoted {}",
6554 r + 1,
6555 rv.vote.label()
6556 ),
6557 );
6558 }
6559 Err(e) => {
6560 rec.failed = Some(e.to_string());
6561 self.state.event(
6562 "review",
6563 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6564 );
6565 }
6566 }
6567 reconsideration.push(rec);
6568 }
6569 } else if initial_votes.len() > 1 {
6570 self.state.event(
6571 "review",
6572 format!(
6573 "round {round}: votes agreed ({}) — no reconsideration",
6574 initial_votes[0].label()
6575 ),
6576 );
6577 }
6578
6579 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6580 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6581 let defer_e2e =
6592 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6593 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6594 let reason =
6595 format!("{blocking} blocking finding(s) already required a fix this round");
6596 self.state.event(
6597 "verify",
6598 format!(
6599 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6600 {}); it will run once a round has none left",
6601 short(&head)
6602 ),
6603 );
6604 (Vec::new(), false, true, Some(reason))
6605 } else {
6606 let e2e_commands = self.state.config.verify.e2e.clone();
6607 let cache_dir = self.state.config.cache_dir();
6608 let context = format!("round {round}");
6609 let wait = self.repository_wait();
6610 let (e2e, verify_retried) = with_cache_lease(
6611 &mut self.state,
6612 cache_dir.as_deref(),
6613 "e2e",
6614 "e2e",
6615 &winner.worktree,
6616 &head,
6617 verify_timeout,
6618 &context,
6619 &wait,
6620 |state, budget| {
6621 let shell = shell.clone();
6622 let e2e_commands = e2e_commands.clone();
6623 let worktree = winner.worktree.clone();
6624 let context = context.clone();
6625 async move {
6626 run_e2e_with_retry(
6627 state,
6628 &shell,
6629 &e2e_commands,
6630 &worktree,
6631 budget,
6632 &context,
6633 )
6634 .await
6635 }
6636 },
6637 )
6638 .await?;
6639 (e2e, verify_retried, false, None)
6640 };
6641
6642 let expected = records.len();
6643 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6644 let incomplete = answered < expected;
6645 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6646 let policy = self.state.config.graph.incomplete_review;
6647 let clean = round_is_clean(
6648 blocking,
6649 e2e_ok,
6650 answered,
6651 expected,
6652 round_quota_missing,
6653 policy,
6654 );
6655
6656 let mut round_record = ReviewRound {
6657 round,
6658 head: head.clone(),
6659 verified_head: None,
6660 verified_at: None,
6661 reviews: records,
6662 e2e,
6663 verify_retried,
6664 e2e_deferred,
6665 e2e_defer_reason,
6666 fix: None,
6667 blocking,
6668 answered,
6669 expected,
6670 clean,
6671 progressed: false,
6672 vote_split,
6673 reconsideration,
6674 verdict: None,
6675 };
6676 round_record.verdict = ReviewVote::worst(
6680 round_record
6681 .final_votes()
6682 .into_iter()
6683 .map(|(_, _, vote)| vote),
6684 );
6685 if !matches!(
6696 round_record.e2e_status(),
6697 E2eStatus::Deferred | E2eStatus::NotConfigured
6698 ) {
6699 round_record.verified_head = Some(head.clone());
6700 round_record.verified_at = Some(Timestamp::now());
6701 }
6702
6703 if incomplete {
6704 let missing: Vec<String> = round_record
6705 .reviews
6706 .iter()
6707 .filter(|r| r.failed.is_some())
6708 .map(|r| format!("review-{}", r.reviewer))
6709 .collect();
6710 self.state.event(
6711 "review",
6712 format!(
6713 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6714 missing.join(", ")
6715 ),
6716 );
6717 }
6718
6719 if clean {
6720 self.state.event(
6721 "review",
6722 if incomplete && policy == IncompleteReviewPolicy::Warn {
6723 format!(
6724 "round {round}: clean (warn policy, incomplete panel) — no \
6725 blocking findings from the seats that answered, verification green"
6726 )
6727 } else if incomplete {
6728 format!(
6729 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6730 quorum) — no blocking findings from the seats that answered, \
6731 verification green",
6732 expected - answered
6733 )
6734 } else {
6735 format!("round {round}: clean — no blocking findings, verification green")
6736 },
6737 );
6738 self.state.reviews.push(round_record);
6739 self.state.status = RunStatus::Gating;
6740 self.state.save()?;
6741 return Ok(());
6742 }
6743
6744 if incomplete && blocking == 0 && e2e_ok {
6752 self.state.reviews.push(round_record);
6753 self.state.save()?;
6754 if round == max_rounds {
6755 self.state.status = RunStatus::Blocked;
6756 self.state.event(
6757 "review",
6758 format!(
6759 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6760 refusing to call it clean",
6761 expected - answered
6762 ),
6763 );
6764 return Ok(());
6765 }
6766 continue;
6767 }
6768
6769 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6781 self.state.reviews.push(round_record);
6782 return self
6783 .stop_unresolved_review(
6784 "the round's own verification could not run",
6785 &shell,
6786 &winner.worktree,
6787 )
6788 .await;
6789 }
6790
6791 if round == max_rounds {
6792 self.state.reviews.push(round_record);
6793 return self
6794 .stop_reviewing(
6795 &format!(
6796 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6797 ),
6798 &shell,
6799 &winner.worktree,
6800 )
6801 .await;
6802 }
6803
6804 if !self.fix_review_round(&winner, round_record).await? {
6805 return Ok(());
6806 }
6807 }
6808 Ok(())
6809 }
6810
6811 async fn fix_review_round(
6814 &mut self,
6815 winner: &Candidate,
6816 mut round_record: ReviewRound,
6817 ) -> Result<bool> {
6818 let round = round_record.round;
6819 let shell = self.state.config.shell();
6820 let step = self.run_fix_round(winner, &mut round_record).await?;
6821 if matches!(step, FixStep::Resource) {
6822 self.stop_for_resource()?;
6823 return Ok(false);
6824 }
6825 if self.state.reviews.len() < round {
6826 self.state.reviews.push(round_record);
6827 } else {
6828 self.state.reviews[round - 1] = round_record;
6829 }
6830 self.state.save()?;
6831
6832 if matches!(step, FixStep::Recorded { report_lost: true }) {
6845 self.stop_reviewing(
6846 "the fixer's adoption report never came back, even after resuming its \
6847 own seat; refusing to start another round against the same worktree \
6848 while that is unresolved",
6849 &shell,
6850 &winner.worktree,
6851 )
6852 .await?;
6853 return Ok(false);
6854 }
6855
6856 let streak = self
6857 .state
6858 .reviews
6859 .iter()
6860 .rev()
6861 .take_while(|r| !r.progressed)
6862 .count();
6863 if streak >= STAGNANT_LIMIT {
6864 self.stop_reviewing(
6865 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6866 &shell,
6867 &winner.worktree,
6868 )
6869 .await?;
6870 return Ok(false);
6871 }
6872 Ok(true)
6873 }
6874
6875 async fn run_fix_round(
6877 &mut self,
6878 winner: &Candidate,
6879 round_record: &mut ReviewRound,
6880 ) -> Result<FixStep> {
6881 let round = round_record.round;
6882 let max_rounds = self.state.config.graph.review_rounds;
6883 let run_id = self.state.id.clone();
6884 let prompts = self.state.config.prompts.clone();
6885 let language = self.state.config.graph.language.clone();
6886 let sessions = self.state.config.graph.sessions;
6887 let artifacts = self.merge_repair_artifacts();
6888 let base = self.landing_base();
6889 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6890 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6891 let this_round_verification = round_record.verification_summary(&head);
6892 let all_findings: Vec<_> = round_record
6893 .reviews
6894 .iter()
6895 .flat_map(|r| r.findings.iter().cloned())
6896 .collect();
6897 let blocking_findings: Vec<_> = all_findings
6900 .iter()
6901 .filter(|f| f.severity.blocks())
6902 .cloned()
6903 .collect();
6904 let fix_prompt = prompt::fix_in(
6905 &self.state.instruction,
6906 &blocking_findings,
6907 this_round_verification.as_ref(),
6908 round,
6909 max_rounds,
6910 &language,
6911 &self.state.config.graph.github_language,
6912 );
6913 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6914 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6915 let (job, seat, out) = self
6916 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6917 prompt: fix_prompt.clone(),
6918 spec,
6919 seat,
6920 cwd: winner.worktree.clone(),
6921 timeout,
6922 allow_write: true,
6923 sessions,
6924 artifacts: artifacts.clone(),
6925 stem: format!("fix-{round}"),
6926 handover: None,
6927 })
6928 .await;
6929 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6932 return Ok(FixStep::Resource);
6933 }
6934 let agent_id = seat.agent.clone();
6935
6936 let mut fix = FixRecord {
6937 agent: agent_id,
6938 addressed: Vec::new(),
6939 rejected: Vec::new(),
6940 notes: String::new(),
6941 committed: false,
6942 failed: None,
6943 duration_ms: 0,
6944 continuation: None,
6945 };
6946 let mut continuation = ContinuationRecord::not_needed();
6947 let mut final_seat = seat.clone();
6948 match out {
6949 AgentOutcome::Ok(o) => {
6950 fix.duration_ms = o.duration_ms;
6951 let parsed = verdict::extract_json::<FixReport>(&o.text);
6952 let incomplete_reason = match &parsed {
6959 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6960 "the reply parsed, but it reported a command whose own CLI never \
6961 confirmed an exit status"
6962 .to_owned(),
6963 ),
6964 Ok(_) => None,
6965 Err(e) => Some(e.to_string()),
6966 };
6967 match incomplete_reason {
6968 None => {
6969 let report = parsed.expect("checked Ok above");
6970 fix.addressed = report.addressed;
6971 fix.rejected = report.rejected;
6972 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6973 }
6974 Some(reason) => {
6975 let (resumed_seat, resolved, failure, cont) = self
6976 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6977 .await;
6978 fix.duration_ms += cont.cumulative_wait_ms;
6979 continuation = cont;
6980 final_seat = resumed_seat;
6981 match resolved {
6982 Some(report) => {
6983 fix.addressed = report.addressed;
6984 fix.rejected = report.rejected;
6985 fix.notes =
6986 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6987 }
6988 None => fix.failed = failure,
6989 }
6990 }
6991 }
6992 }
6993 AgentOutcome::Dropped(o) => {
6995 fix.duration_ms = o.duration_ms;
6996 let why = o
6997 .dropped
6998 .as_ref()
6999 .map(|d| d.why.as_str())
7000 .unwrap_or("the CLI ended the stream without delivering its answer");
7001 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7002 }
7003 AgentOutcome::Quota(o) => {
7004 self.state.quota.push(QuotaLoss {
7005 seat: final_seat.key.clone(),
7006 node: "fix".to_owned(),
7007 at: Timestamp::now(),
7008 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7009 });
7010 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7011 }
7012 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7013 }
7014 fix.continuation = Some(continuation);
7015 self.state.seats.insert(final_seat.key.clone(), final_seat);
7016 let coordination = self.coordinate(&winner.worktree).await?;
7017 if let Ok(r) = git::rescue_commit(
7018 &winner.worktree,
7019 &format!("magi: review round {round} fixes (uncommitted work)"),
7020 )
7021 .await
7022 {
7023 self.state.note_withheld("fix", &r.withheld);
7024 }
7025 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7026 fix.committed = after != before;
7027 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7035 drop(coordination);
7036 let progressed = diff_after != patch;
7037 let commit_note = if fix.committed {
7038 "committed"
7039 } else {
7040 "NO new commit"
7041 };
7042 let tree_note = if progressed {
7043 "changed vs base"
7044 } else {
7045 "unchanged vs base"
7046 };
7047 self.state.event(
7048 "fix",
7049 match &fix.failed {
7050 Some(reason) => {
7056 format!(
7057 "round {round}: fixer's adoption report was lost ({reason}); \
7058 {commit_note}, tree {tree_note}"
7059 )
7060 }
7061 None => format!(
7062 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7063 {tree_note}{}",
7064 fix.addressed.len(),
7065 fix.rejected.len(),
7066 if continuation.outcome == ContinuationOutcome::Resumed {
7067 format!(
7068 " (adoption report recovered after {} continuation(s))",
7069 continuation.attempts
7070 )
7071 } else {
7072 String::new()
7073 },
7074 ),
7075 },
7076 );
7077 round_record.fix = Some(fix);
7078 round_record.progressed = progressed;
7079 Ok(FixStep::Recorded {
7080 report_lost: matches!(
7081 continuation.outcome,
7082 ContinuationOutcome::Exhausted
7083 | ContinuationOutcome::QuotaLost
7084 | ContinuationOutcome::NoSession
7085 ),
7086 })
7087 }
7088
7089 fn record_contested_handoff(&mut self) {
7123 if self.state.contested_handoff.is_some() {
7124 return;
7125 }
7126 let Some(contested) = self
7127 .state
7128 .reviews
7129 .last()
7130 .and_then(ReviewRound::contested_handoff)
7131 else {
7132 return;
7133 };
7134 self.state.event(
7135 "review",
7136 format!(
7137 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7138 wait for the owner's approval",
7139 contested.findings.len(),
7140 contested.rejecters.len()
7141 ),
7142 );
7143 self.state.contested_handoff = Some(contested);
7144 }
7145
7146 async fn stop_unresolved_review(
7147 &mut self,
7148 why: &str,
7149 shell: &[String],
7150 worktree: &Path,
7151 ) -> Result<()> {
7152 self.stop_reviewing_inner(why, shell, worktree, true).await
7153 }
7154
7155 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7156 self.stop_reviewing_inner(why, shell, worktree, false).await
7157 }
7158
7159 async fn stop_reviewing_inner(
7160 &mut self,
7161 why: &str,
7162 shell: &[String],
7163 worktree: &Path,
7164 repairable: bool,
7165 ) -> Result<()> {
7166 let round_idx = self.state.reviews.len() - 1;
7167 let needs_catchup_run = matches!(
7175 self.state.reviews[round_idx].e2e_status(),
7176 E2eStatus::Deferred | E2eStatus::ResourceBlocked
7177 );
7178 if needs_catchup_run {
7179 let round = self.state.reviews[round_idx].round;
7180 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7181 let commands = self.state.config.verify.e2e.clone();
7182 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7183 let cache_dir = self.state.config.cache_dir();
7184 let context = format!(
7185 "round {round}: verification unresolved, catching up before the final decision"
7186 );
7187 let wait = self.repository_wait();
7188 let (outcomes, verify_retried) = with_cache_lease(
7189 &mut self.state,
7190 cache_dir.as_deref(),
7191 "e2e",
7192 "e2e",
7193 worktree,
7194 &attempted_head,
7195 timeout,
7196 &context,
7197 &wait,
7198 |state, budget| {
7199 let shell = shell.to_vec();
7200 let commands = commands.clone();
7201 let context = context.clone();
7202 async move {
7203 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7204 .await
7205 }
7206 },
7207 )
7208 .await?;
7209 let last = &mut self.state.reviews[round_idx];
7210 last.e2e = outcomes;
7211 last.verify_retried = verify_retried;
7212 last.verified_head = Some(attempted_head);
7219 last.verified_at = Some(Timestamp::now());
7220 if verify_inconclusive(&last.e2e) {
7221 self.state.save()?;
7228 return Ok(());
7229 }
7230 last.e2e_deferred = false;
7231 }
7232 let last = &self.state.reviews[round_idx];
7233 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7234
7235 match last.e2e_status() {
7236 E2eStatus::Failed => {
7237 let red: Vec<String> = last
7238 .e2e
7239 .iter()
7240 .filter(|o| !o.ok())
7241 .map(|o| {
7242 format!(
7243 "`{}` -> {:?}\n{}",
7244 o.command,
7245 o.code,
7246 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7247 )
7248 })
7249 .collect();
7250 let can_repair = last.blocking == 0 && last.fix.is_none();
7251 self.state.reviews[round_idx].clean = false;
7252 self.state
7253 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7254 self.state.status = RunStatus::Blocked;
7255 if repairable
7256 && can_repair
7257 && self.state.reviews.len() < self.state.config.graph.review_rounds
7258 {
7259 self.state.status = RunStatus::Reviewing;
7260 }
7261 }
7262 E2eStatus::ResourceBlocked => {
7267 self.state.event(
7268 "review",
7269 format!(
7270 "{why}; e2e could not run (resource unavailable); not \
7271 deciding yet"
7272 ),
7273 );
7274 }
7275 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7276 self.state.event(
7277 "review",
7278 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7279 );
7280 self.record_contested_handoff();
7281 self.state.status = RunStatus::Gating;
7282 }
7283 }
7284 self.state.save()?;
7285 Ok(())
7286 }
7287
7288 async fn gate(&mut self) -> Result<()> {
7291 if self.hold_base_sync().await? {
7292 return Ok(());
7293 }
7294 if self.state.status == RunStatus::Failed
7306 || self
7307 .state
7308 .base_sync
7309 .as_ref()
7310 .is_some_and(|s| s.conflict.is_some())
7311 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7312 != Some(RunStatus::Gating)
7313 {
7314 return Ok(());
7315 }
7316 if self.state.gate_ran {
7317 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7328 self.state.status = RunStatus::Blocked;
7329 self.state.save()?;
7330 }
7331 return Ok(());
7332 }
7333 let Some(winner) = self.state.winner().cloned() else {
7334 return Ok(());
7335 };
7336 self.state.status = RunStatus::Gating;
7337 let mut outcomes = self.run_gate(&winner).await?;
7338 loop {
7339 if verify_inconclusive(&outcomes) {
7350 self.state.save()?;
7351 return Ok(());
7352 }
7353 if outcomes.iter().all(CommandOutcome::ok) {
7354 break;
7355 }
7356 match self.gate_fix_round(&winner, &outcomes).await? {
7357 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7358 GateFix::Stop => break,
7359 GateFix::Defer => {
7360 self.state.save()?;
7361 return Ok(());
7362 }
7363 }
7364 }
7365 let passed = outcomes.iter().all(CommandOutcome::ok);
7366 self.state.gate = outcomes;
7367 self.state.gate_ran = true;
7368 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7369 if !passed {
7370 self.state.status = RunStatus::Blocked;
7371 let spent = self.state.gate_fixes.len();
7372 self.state.event(
7373 "gate",
7374 if spent == 0 {
7375 "gate failed; not merging".to_owned()
7376 } else {
7377 format!("gate failed after {spent} gate-fix round(s); not merging")
7378 },
7379 );
7380 }
7381 self.state.save()?;
7382 Ok(())
7383 }
7384
7385 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7395 let commands = state.config.verify.pre_gate.clone();
7396 if commands.is_empty() {
7397 return;
7398 }
7399 let shell = state.config.shell();
7400 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7401 let (outcomes, _) = run_commands(
7402 state,
7403 "pre_gate",
7404 "pre_gate",
7405 0,
7406 &shell,
7407 &commands,
7408 &winner.worktree,
7409 timeout,
7410 )
7411 .await;
7412 for o in &outcomes {
7413 if !o.ok() {
7414 tracing::warn!(
7415 "pre_gate `{}` failed ({:?}); the gate decides",
7416 o.command,
7417 o.code
7418 );
7419 }
7420 state.event(
7421 "pre_gate",
7422 format!(
7423 "`{}` -> {}",
7424 o.command,
7425 if o.ok() {
7426 "pass".to_owned()
7427 } else {
7428 format!(
7429 "FAIL ({:?})\n{}",
7430 o.code,
7431 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7432 )
7433 }
7434 ),
7435 );
7436 }
7437 state.pre_gate = outcomes;
7438 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7439 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7440 Ok(head) => {
7441 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7442 state.pre_gate_commit = Some(head);
7443 }
7444 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7445 },
7446 Ok(false) => {}
7447 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7448 }
7449 if let Err(e) = state.save() {
7450 tracing::warn!("could not persist the pre_gate record: {e:#}");
7451 }
7452 }
7453
7454 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7457 let wait = self.repository_wait();
7458 run_landing_gate(&mut self.state, winner, &wait).await
7459 }
7460
7461 async fn gate_fix_round(
7474 &mut self,
7475 winner: &Candidate,
7476 outcomes: &[CommandOutcome],
7477 ) -> Result<GateFix> {
7478 let cap = self.state.config.graph.gate_fix_rounds;
7479 let spent = self.state.gate_fixes.len();
7480 if spent >= cap {
7481 if cap > 0 {
7482 self.state.event(
7483 "gate",
7484 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7485 );
7486 }
7487 return Ok(GateFix::Stop);
7488 }
7489 if !gate_fixable(outcomes) {
7490 self.state.event(
7491 "gate",
7492 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7493 command or similar); not spending a fix round on it",
7494 );
7495 return Ok(GateFix::Stop);
7496 }
7497 let min_free = self.state.config.disk.min_free_bytes;
7498 if min_free > 0 {
7499 match crate::disk::free_bytes(&winner.worktree) {
7500 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7501 Ok(free) => {
7502 self.state.event(
7503 "gate",
7504 format!(
7505 "only {free} bytes free ({min_free} required by `[disk] \
7506 min_free_bytes`); not spending a fix round on a failure the disk \
7507 may explain"
7508 ),
7509 );
7510 return Ok(GateFix::Stop);
7511 }
7512 Err(e) => {
7513 self.state.event(
7514 "gate",
7515 format!("free disk space could not be measured ({e:#}); no fix round"),
7516 );
7517 return Ok(GateFix::Stop);
7518 }
7519 }
7520 }
7521
7522 let attempt = spent + 1;
7523 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7524 let base = self.landing_base();
7525 let fix_prompt = prompt::gate_fix_in(
7526 &self.state.instruction,
7527 &failed,
7528 attempt,
7529 cap,
7530 &self.state.config.graph.language,
7531 &self.state.config.graph.github_language,
7532 );
7533 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7534 let sessions = self.state.config.graph.sessions;
7535 let artifacts = self.merge_repair_artifacts();
7536 self.state.event(
7537 "gate",
7538 format!("gate failed; gate-fix round {attempt} of {cap}"),
7539 );
7540 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7541 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7542 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7545 head: before.clone(),
7546 passed: false,
7547 outcomes: Vec::new(),
7548 });
7549 self.state.save()?;
7550 let (_, seat, out) = self
7551 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7552 prompt: fix_prompt.clone(),
7553 spec,
7554 seat,
7555 cwd: winner.worktree.clone(),
7556 timeout,
7557 allow_write: true,
7558 sessions,
7559 artifacts: artifacts.clone(),
7560 stem: format!("gate-fix-{attempt}"),
7561 handover: None,
7562 })
7563 .await;
7564 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7567 return Ok(GateFix::Defer);
7568 }
7569 let mut record = GateFixRecord {
7570 agent: seat.agent.clone(),
7571 failed,
7572 notes: String::new(),
7573 committed: false,
7574 error: None,
7575 };
7576 match out {
7577 AgentOutcome::Ok(o) => {
7578 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7581 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7582 }
7583 }
7584 AgentOutcome::Dropped(_) => {
7585 record.error = Some("the CLI dropped the stream".to_owned());
7586 }
7587 AgentOutcome::Quota(o) => {
7588 self.state.quota.push(QuotaLoss {
7589 seat: seat.key.clone(),
7590 node: "gate-fix".to_owned(),
7591 at: Timestamp::now(),
7592 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7593 });
7594 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7595 }
7596 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7597 }
7598 self.state.seats.insert(seat.key.clone(), seat);
7599 let coordination = self.coordinate(&winner.worktree).await?;
7600 if let Ok(r) = git::rescue_commit(
7601 &winner.worktree,
7602 &format!("magi: gate fix {attempt} (uncommitted work)"),
7603 )
7604 .await
7605 {
7606 self.state.note_withheld("gate-fix", &r.withheld);
7607 }
7608 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7609 record.committed = after != before;
7610 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7611 let note = record.error.clone();
7612 self.state.gate_fixes.push(record);
7613 self.state.save()?;
7614 if !changed {
7615 self.state.event(
7616 "gate",
7617 match note {
7618 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7619 None => format!("gate-fix round {attempt}: the tree did not change"),
7620 },
7621 );
7622 return Ok(GateFix::Stop);
7623 }
7624 self.state.event(
7625 "gate",
7626 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7627 );
7628
7629 drop(coordination);
7630 let wait = self.repository_wait();
7631 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7632 }
7633
7634 async fn guarded_pr_message(
7641 &mut self,
7642 winner: &Candidate,
7643 facts: Option<&BranchFacts>,
7644 posting: bool,
7645 ) -> Result<Option<PrMessage>> {
7646 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7647 let head = match final_head.as_deref() {
7648 Some(h) => crate::pr_outcome::FinalHead::At(h),
7649 None => crate::pr_outcome::FinalHead::Unknown,
7650 };
7651 let summarized = if posting {
7653 Some(self.summarize_change(winner, final_head.as_deref()).await)
7654 } else {
7655 None
7656 };
7657 if self.state.resource_block.is_some() {
7662 self.state.event(
7663 "pr-summary",
7664 "a shared resource was busy; nothing was published, resume to summarize again",
7665 );
7666 return Ok(None);
7667 }
7668 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7669 anyhow::bail!(
7670 "the final commit moved during public description generation; retry before publishing"
7671 );
7672 }
7673 let note = match (&summarized, &self.state.change_summary) {
7674 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7675 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7676 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7677 (None, _) => ChangeNote::NotAsked,
7678 };
7679 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7680 if !posting {
7681 let identity = crate::scrub::Identity::current();
7682 return Ok(Some(PrMessage {
7683 title: crate::scrub::scrub(&pr.title, &identity),
7684 body: crate::scrub::scrub(&pr.body, &identity),
7685 }));
7686 }
7687 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7690 if let Some(known) = self
7691 .state
7692 .github_text
7693 .clone()
7694 .filter(|g| g.fingerprint == raw_fp)
7695 {
7696 return self.settle_withheld(winner, &pr, known).await;
7697 }
7698 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7699 let mut violations =
7700 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7701 let identity = crate::scrub::Identity::current();
7702 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7703 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7704 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7705 {
7706 violations.push(crate::github_text::Violation::SensitiveData);
7707 }
7708 let has_language = violations
7712 .iter()
7713 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7714 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7715 self.state.event(
7716 "github-text",
7717 if has_language {
7718 format!("description rejected: {violations:?}; requesting one rewrite")
7719 } else {
7720 format!("description rejected: {violations:?}; asking the owner")
7721 },
7722 );
7723 let mut rewritten = false;
7724 let mut rejected_rewrite = None;
7725 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7726 {
7727 let output = text;
7728 if let Some(title) = summary_title(&output) {
7729 let raw_body = summary_without_title(&output);
7732 let clean_title = crate::scrub::scrub(&title, &identity);
7733 let mut body = crate::scrub::scrub(&raw_body, &identity);
7734 let redacted = clean_title != title || body != raw_body;
7735 let title = clean_title;
7736 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7737 if !complete {
7738 self.state.event(
7739 "github-text",
7740 "description rewrite omitted required findings",
7741 );
7742 }
7743 if !body.trim().is_empty() {
7744 body.push_str("\n\n");
7745 body.push_str(&public_pr_evidence(&self.state, head));
7746 }
7747 let marker = format!("magi:run/{}", self.state.id);
7748 if !body.trim().is_empty() && !body.contains(&marker) {
7749 body.push_str(&format!("\n\n{marker}"));
7750 }
7751 let rewrite = PrMessage {
7752 title: title.clone(),
7753 body: body.clone(),
7754 };
7755 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7756 let mut failed = crate::github_text::check_for(
7757 &self.state.config.graph,
7758 &title,
7759 &body,
7760 redecision,
7761 );
7762 if (!crate::github_text::shareable(&title)
7763 || !crate::github_text::shareable(&body))
7764 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7765 {
7766 failed.push(crate::github_text::Violation::SensitiveData);
7767 }
7768 if complete
7769 && !title.trim().is_empty()
7770 && !body.trim().is_empty()
7771 && !failed.is_empty()
7772 {
7773 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7774 title: title.clone(),
7775 body: body.clone(),
7776 failed: failed
7777 .iter()
7778 .map(|v| crate::github_text::category(*v).to_owned())
7779 .collect(),
7780 });
7781 }
7782 if complete
7783 && !title.trim().is_empty()
7784 && !raw_body.trim().is_empty()
7785 && failed.is_empty()
7786 {
7787 decision = redecision;
7788 pr = PrMessage { title, body };
7789 rewritten = true;
7790 self.state
7791 .event("github-text", "description rewrite passed");
7792 if redacted {
7793 self.state.event(
7794 "github-text",
7795 "sensitive data removed from description rewrite",
7796 );
7797 }
7798 }
7799 }
7800 }
7801 if !rewritten {
7802 self.state.event(
7803 "github-text",
7804 "description rewrite unavailable or rejected; asking the owner",
7805 );
7806 if let Some(w) =
7807 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7808 {
7809 self.retire_github_text_question("the withheld text changed");
7812 let record = crate::run::GithubTextAsk {
7813 rewrite: rejected_rewrite,
7814 fingerprint: raw_fp,
7815 title: w.title,
7816 body: w.body,
7817 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7818 question: None,
7819 asks: 0,
7820 resolved: false,
7821 chosen_title: None,
7822 choice: None,
7823 };
7824 return self.settle_withheld(winner, &pr, record).await;
7825 }
7826 }
7827 }
7828 let (title, body) =
7829 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7830 Ok(Some(PrMessage { title, body }))
7831 }
7832
7833 async fn summarize_change(
7844 &mut self,
7845 winner: &Candidate,
7846 head: Option<&str>,
7847 ) -> std::result::Result<(), String> {
7848 let Some(head) = head else {
7849 return Err("the final commit could not be read".to_owned());
7850 };
7851 let remote = self.state.config.merge.remote.clone();
7852 let tracking = format!("{remote}/{}", self.state.base_branch);
7853 if !git::rev_exists(&winner.worktree, &tracking).await {
7854 return Err(format!("the remote base `{tracking}` could not be read"));
7855 }
7856 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7857 Ok(b) if !b.is_empty() => b,
7858 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7859 };
7860 let evidence = crate::pr_outcome::summary_evidence(
7861 &self.state,
7862 crate::pr_outcome::FinalHead::At(head),
7863 );
7864 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7865 if self
7866 .state
7867 .change_summary
7868 .as_ref()
7869 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7870 {
7871 return Ok(());
7872 }
7873 if self.state.change_summary.take().is_some() {
7874 self.state.event(
7875 "pr-summary",
7876 "the final diff or evidence changed; the earlier change summary was discarded",
7877 );
7878 }
7879 if let Err(e) = write_artifact(
7881 &self.state,
7882 &format!("public-description-{fingerprint}.md"),
7883 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7884 ) {
7885 tracing::warn!("could not save public description evidence: {e:#}");
7886 }
7887 let chain = match agent::pick_chain(
7888 &self.state.config.agents,
7889 self.state.config.roles.synthesizer.as_ref(),
7890 &agent::installed,
7891 "synthesizer",
7892 ) {
7893 Ok(chain) => chain,
7894 Err(e) => {
7895 tracing::warn!("no agent for the change summary: {e:#}");
7896 self.state.event("pr-summary", "no summary agent available");
7897 return Err("no agent was available to summarize it".to_owned());
7898 }
7899 };
7900 let subjects = git::subjects(&winner.worktree, &base, head)
7901 .await
7902 .unwrap_or_default();
7903 let stat = git::diff_stat(&winner.worktree, &base, head)
7904 .await
7905 .unwrap_or_default();
7906 let stat: String = stat.chars().take(6_000).collect();
7907 let mut prompt = crate::prompt::change_summary_in(
7908 &base,
7909 head,
7910 &subjects,
7911 stat.trim(),
7912 &self.state.config.graph.github_language,
7913 );
7914 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7915 let sessions = self.state.config.graph.sessions;
7916 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7917 let prompts = self.state.config.prompts.clone();
7918 let cache = self.state.config.cache_dir();
7919 let run = self.state.id.clone();
7920 for (i, spec) in chain.into_iter().enumerate() {
7921 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7922 let github_spec = match agent::for_github_text(
7923 &spec,
7924 &self.state.config.graph.github_language,
7925 &winner.worktree,
7926 ) {
7927 Ok(spec) => spec,
7928 Err(e) => {
7929 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7930 continue;
7931 }
7932 };
7933 let job = SeatJob {
7934 spec: github_spec,
7935 seat,
7936 cwd: winner.worktree.clone(),
7937 prompt: prompt.clone(),
7938 timeout,
7939 allow_write: false,
7940 sessions,
7941 artifacts: agent::artifacts_dir(&self.state.dir()),
7942 stem: if i == 0 {
7943 "change-summary".to_owned()
7944 } else {
7945 format!("change-summary-{}", spec.id)
7946 },
7947 handover: None,
7948 };
7949 let ctx = WaveCtx {
7950 run: &run,
7951 node: "pr-summary",
7952 prompts: &prompts,
7953 cache: cache.as_deref(),
7954 round: None,
7955 carry_seats: false,
7956 pauses: Some((&self.pause, &self.interrupt)),
7957 };
7958 let (seat, outcome) =
7959 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7960 self.state.seats.insert(seat.key.clone(), seat);
7961 let text = match outcome {
7962 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7963 AgentOutcome::ResourceBlocked(why) => {
7964 if self.state.resource_block.is_none() {
7967 self.state.resource_block = Some(crate::run::ResourceBlock {
7968 cause: crate::run::ResourceBlockCause::Other,
7969 node: "pr-summary".to_owned(),
7970 kind: "shared resource".to_owned(),
7971 detail: why,
7972 at: Timestamp::now(),
7973 });
7974 }
7975 return Err("a shared resource was busy".to_owned());
7976 }
7977 _ => continue,
7978 };
7979 let blind = &self.state.config.blind;
7980 let Some(title) = summary_title(&text) else {
7981 continue;
7982 };
7983 let body: String = summary_without_title(&text)
7984 .lines()
7985 .filter(|l| !l.contains("magi:run/"))
7986 .collect::<Vec<_>>()
7987 .join("\n");
7988 let body = blind::sanitize_prose(body.trim(), blind);
7989 let title = blind::sanitize_prose(&title, blind);
7990 if body.trim().is_empty() || title.trim().is_empty() {
7991 continue;
7992 }
7993 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7994 continue;
7995 }
7996 if git::rev_parse(&winner.worktree, "HEAD")
7998 .await
7999 .ok()
8000 .as_deref()
8001 != Some(head)
8002 {
8003 return Err("the branch moved while the summary was written".to_owned());
8004 }
8005 if git::merge_base(&winner.worktree, &tracking, head)
8006 .await
8007 .ok()
8008 .as_deref()
8009 != Some(base.as_str())
8010 || crate::pr_outcome::summary_evidence(
8011 &self.state,
8012 crate::pr_outcome::FinalHead::At(head),
8013 ) != evidence
8014 {
8015 return Err("the summary evidence changed while it was written".to_owned());
8016 }
8017 self.state.change_summary = Some(crate::run::ChangeSummary {
8018 head: head.to_owned(),
8019 base,
8020 evidence: fingerprint,
8021 title,
8022 body,
8023 });
8024 self.state.event(
8025 "pr-summary",
8026 format!("change summary written by {} for the final diff", spec.id),
8027 );
8028 if let Err(e) = self.state.save() {
8029 tracing::warn!("could not save the change summary: {e:#}");
8030 }
8031 return Ok(());
8032 }
8033 self.state
8034 .event("pr-summary", "no agent produced a change summary");
8035 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8036 }
8037
8038 async fn rewrite_pr_text(
8048 &mut self,
8049 winner: &Candidate,
8050 pr: &PrMessage,
8051 violations: &[crate::github_text::Violation],
8052 ) -> Option<String> {
8053 let prompt = format!(
8054 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8055 pr.title, pr.body
8056 );
8057 let sessions = self.state.config.graph.sessions;
8058 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8059 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8062 let key = format!("impl-{}", winner.label);
8063 if let Some(spec) = self
8064 .state
8065 .config
8066 .agents
8067 .iter()
8068 .find(|a| a.id == winner.agent)
8069 .cloned()
8070 && let Some(existing) = self.state.seats.get(&key)
8071 && existing.agent == spec.id
8072 && agent::has_session(spec.kind, existing, sessions)
8073 {
8074 attempts.push((spec, Some(existing.clone())));
8075 }
8076 match agent::pick_chain(
8077 &self.state.config.agents,
8078 self.state.config.roles.language_judge.as_ref(),
8079 &agent::installed,
8080 "language_judge",
8081 ) {
8082 Ok(chain) => {
8083 for spec in chain {
8084 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8085 continue;
8086 }
8087 attempts.push((spec, None));
8088 }
8089 }
8090 Err(e) if attempts.is_empty() => {
8091 tracing::warn!("no agent for the description rewrite: {e:#}");
8092 self.state
8093 .event("github-text", "no rewrite agent available");
8094 return None;
8095 }
8096 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8097 }
8098 let prompts = self.state.config.prompts.clone();
8099 let cache = self.state.config.cache_dir();
8100 let run = self.state.id.clone();
8101 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8102 let resumed = resumed_seat.is_some();
8103 let seat = resumed_seat.unwrap_or_else(|| {
8104 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8105 });
8106 let github_spec = match agent::for_github_text(
8107 &spec,
8108 &self.state.config.graph.github_language,
8109 &winner.worktree,
8110 ) {
8111 Ok(spec) => spec,
8112 Err(e) => {
8113 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8114 continue;
8115 }
8116 };
8117 let job = SeatJob {
8118 spec: github_spec,
8119 seat,
8120 cwd: winner.worktree.clone(),
8121 prompt: prompt.clone(),
8122 timeout: retry_budget(timeout_full, resumed),
8123 allow_write: false,
8124 sessions,
8125 artifacts: agent::artifacts_dir(&self.state.dir()),
8126 stem: if i == 0 {
8127 "github-text-rewrite".to_owned()
8128 } else {
8129 format!("github-text-rewrite-{}", spec.id)
8130 },
8131 handover: None,
8132 };
8133 let ctx = WaveCtx {
8134 run: &run,
8135 node: "github-text",
8136 prompts: &prompts,
8137 cache: cache.as_deref(),
8138 round: None,
8139 carry_seats: false,
8140 pauses: Some((&self.pause, &self.interrupt)),
8141 };
8142 let (seat, outcome) =
8143 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8144 self.state.seats.insert(seat.key.clone(), seat);
8145 match outcome {
8146 AgentOutcome::Ok(o)
8147 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8148 {
8149 return Some(o.text);
8150 }
8151 AgentOutcome::ResourceBlocked(_) => return None,
8152 _ => {}
8153 }
8154 }
8155 None
8156 }
8157
8158 fn retire_github_text_question(&mut self, why: &str) {
8160 let Some(id) = self
8161 .state
8162 .github_text
8163 .as_ref()
8164 .and_then(|g| g.question.clone())
8165 else {
8166 return;
8167 };
8168 let _ = ask::Questions::open().update(&id, |q| {
8169 q.abandon(why);
8170 Ok(())
8171 });
8172 }
8173
8174 async fn settle_withheld(
8178 &mut self,
8179 winner: &Candidate,
8180 pr: &PrMessage,
8181 mut g: crate::run::GithubTextAsk,
8182 ) -> Result<Option<PrMessage>> {
8183 use crate::github_text as gt;
8184 let store = ask::Questions::open();
8185 let timeout = self.state.config.graph.answer_timeout;
8186 if !g.resolved {
8187 let mut standing = g
8188 .question
8189 .as_ref()
8190 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8191 if let Some(q) = standing.as_ref()
8192 && q.status.open()
8193 && gt::expired(q, timeout)
8194 {
8195 let why = format!("no answer within {}s of asking", timeout.max(1));
8196 standing = store
8197 .update(&q.id, |q| {
8198 q.abandon(&why);
8199 Ok(())
8200 })
8201 .ok()
8202 .map(|(q, ())| q);
8203 }
8204 let mut ask_again: Option<Option<Vec<String>>> = None;
8205 match standing {
8206 None if g.asks <= 1 => {
8209 g.asks = 0;
8210 ask_again = Some(None);
8211 }
8212 None => {
8213 g.resolved = true;
8214 g.chosen_title = None;
8215 g.choice = Some(GithubTextChoice::Fallback);
8216 }
8217 Some(q) if q.status.open() => {
8218 self.park_on_github_text(&g)?;
8219 return Ok(None);
8220 }
8221 Some(q) => match gt::read_reply(&q) {
8222 gt::Reply::Fallback => {
8223 g.resolved = true;
8224 g.chosen_title = None;
8225 g.choice = Some(GithubTextChoice::Fallback);
8226 }
8227 gt::Reply::Original => {
8228 g.resolved = true;
8229 g.chosen_title = None;
8230 g.choice = Some(GithubTextChoice::Original);
8231 }
8232 gt::Reply::Rewrite => {
8233 g.resolved = true;
8234 g.chosen_title = None;
8235 g.choice = Some(if g.rewrite.is_some() {
8236 GithubTextChoice::Rewrite
8237 } else {
8238 GithubTextChoice::Fallback
8239 });
8240 }
8241 gt::Reply::Title(text) => {
8242 let first = text
8243 .lines()
8244 .map(str::trim)
8245 .find(|l| !l.is_empty())
8246 .unwrap_or("");
8247 let decision = if first.is_empty() {
8248 None
8249 } else {
8250 let candidate = PrMessage {
8251 title: first.to_owned(),
8252 body: String::new(),
8253 };
8254 self.judge_pr_language(&winner.worktree, &candidate).await
8255 };
8256 match gt::vet_title(
8257 &text,
8258 decision,
8259 self.state.config.graph.github_english(),
8260 ) {
8261 Ok(title) => {
8262 g.resolved = true;
8263 g.chosen_title = Some(title);
8264 g.choice = Some(GithubTextChoice::MyTitle);
8265 }
8266 Err(cats) if g.asks < 2 => {
8267 ask_again = Some(Some(
8268 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8269 ))
8270 }
8271 Err(_) => {
8272 g.resolved = true;
8273 g.chosen_title = None;
8274 g.choice = Some(GithubTextChoice::Fallback);
8275 }
8276 }
8277 }
8278 },
8279 }
8280 if let Some(rejected) = ask_again {
8281 let w = gt::Withheld {
8284 title: g.title,
8285 body: g.body,
8286 categories: g.categories.clone(),
8287 };
8288 let artifact = match crate::run::write_artifact(
8291 &self.state,
8292 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8293 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8294 ) {
8295 Ok(p) => Some(p),
8296 Err(e) => {
8297 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8298 None
8299 }
8300 };
8301 let (detail, comparison) = gt::question_content(
8302 &w,
8303 &pr.title,
8304 &pr.body,
8305 &self.state.id,
8306 rejected.as_deref(),
8307 artifact.as_deref(),
8308 g.rewrite.as_ref(),
8309 );
8310 let mut q = ask::Question::new(
8311 self.state.id.clone(),
8312 gt::ASK_NODE.to_owned(),
8313 gt::ASK_SEAT.to_owned(),
8314 gt::question_summary(&self.state.config.graph.language, &w),
8315 detail,
8316 gt::question_choices(&w, g.rewrite.as_ref()),
8317 );
8318 q.posting_comparison = Some(comparison);
8319 q.answer_timeout = timeout;
8320 g.question = Some(q.id.clone());
8323 g.asks += 1;
8324 self.state.github_text = Some(g.clone());
8325 self.state.event(
8326 "github-text",
8327 format!("asking the owner about the withheld text ({})", q.short()),
8328 );
8329 self.state.save()?;
8330 store
8331 .put_executable(&mut q)
8332 .context("file the posting-gate question")?;
8333 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8334 tracing::warn!(
8335 "could not notify about posting-gate question {}: {e:#}",
8336 q.short()
8337 );
8338 }
8339 self.park_on_github_text(&g)?;
8340 return Ok(None);
8341 }
8342 }
8343 let resolved = g.chosen_title.clone();
8344 let original = g.choice == Some(GithubTextChoice::Original);
8345 self.state.event(
8347 "github-text",
8348 if original {
8349 format!(
8350 "posting the original text for: {}",
8351 match (g.title, g.body) {
8352 (true, true) => "title, description",
8353 (true, false) => "title",
8354 _ => "description",
8355 }
8356 )
8357 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8358 "posting the saved rewrite title and description".to_owned()
8359 } else if resolved.is_some() {
8360 "posting the owner's replacement title".to_owned()
8361 } else {
8362 "posting the neutral text for the withheld fields".to_owned()
8363 },
8364 );
8365 self.state.github_text = Some(g.clone());
8366 self.state.save()?;
8367 if g.choice == Some(GithubTextChoice::Rewrite)
8368 && let Some(r) = g.rewrite.as_ref()
8369 {
8370 return Ok(Some(PrMessage {
8371 title: r.title.clone(),
8372 body: r.body.clone(),
8373 }));
8374 }
8375 let w = gt::Withheld {
8376 title: g.title,
8377 body: g.body,
8378 categories: g.categories.clone(),
8379 };
8380 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8381 if let Some(t) = resolved {
8382 title = t;
8383 }
8384 let english = gt::LanguageDecision {
8386 title_english: true,
8387 body_english: true,
8388 body_complete: true,
8389 };
8390 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8391 if original {
8392 if g.title {
8395 title = pr.title.clone();
8396 }
8397 if g.body {
8398 body = pr.body.clone();
8399 }
8400 }
8401 Ok(Some(PrMessage { title, body }))
8402 }
8403
8404 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8405 self.state.github_text = Some(g.clone());
8406 self.state.end_pass_parked(ParkCause::Question);
8407 self.state.parked = true;
8408 self.state.event(
8409 "github-text",
8410 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8411 );
8412 self.state.save()
8413 }
8414
8415 async fn judge_pr_language(
8418 &mut self,
8419 cwd: &Path,
8420 pr: &PrMessage,
8421 ) -> Option<crate::github_text::LanguageDecision> {
8422 self.state.config.roles.language_judge.as_ref()?;
8423 if !self.state.config.graph.language_gate() {
8424 return None;
8425 }
8426 let decision =
8427 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8428 self.state.event(
8429 "github-text",
8430 if decision.is_some() {
8431 "language decided by the language judge"
8432 } else {
8433 "language judge unavailable; using built-in heuristics"
8434 },
8435 );
8436 decision
8437 }
8438
8439 async fn merge(&mut self) -> Result<()> {
8440 if self.hold_base_sync().await? {
8441 return Ok(());
8442 }
8443 if self
8458 .state
8459 .base_sync
8460 .as_ref()
8461 .is_some_and(|s| s.conflict.is_some())
8462 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8463 != Some(RunStatus::Gating)
8464 || !self.state.gate_allows_progress()
8472 {
8473 return Ok(());
8474 }
8475 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8484 return Ok(());
8485 }
8486 let Some(winner) = self.state.winner().cloned() else {
8487 return Ok(());
8488 };
8489 let repo = self.state.repo.clone();
8490 let base = self.state.base_branch.clone();
8491 let mode = self.state.config.merge.mode;
8492 let style = self.state.config.merge.style;
8493 let facts = if is_review_run(&self.state) {
8494 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8495 let start = review_base(
8496 &repo,
8497 &self.state.config.merge.remote,
8498 &base,
8499 &self.state.base_commit,
8500 &winner.branch,
8501 )
8502 .await;
8503 branch_facts(&repo, &start, &winner.branch).await
8504 } else {
8505 None
8506 };
8507 let leaked = if is_review_run(&self.state) {
8510 leaked_subjects(&self.state, &winner.branch).await
8511 } else {
8512 Some(Vec::new())
8513 };
8514 let Some(pr) = self
8518 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8519 .await?
8520 else {
8521 return Ok(());
8522 };
8523 let message = pr.commit_message();
8524
8525 let outcome = match mode {
8526 MergeMode::None => MergeOutcome {
8527 mode,
8528 ok: true,
8529 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8530 empty: false,
8531 },
8532 MergeMode::Pr | MergeMode::Local
8533 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8534 {
8535 MergeOutcome {
8536 mode,
8537 ok: false,
8538 detail: empty_candidate_detail(&self.state, &base),
8539 empty: true,
8540 }
8541 }
8542 MergeMode::Local => {
8543 let on = git::current_branch(&repo).await?;
8544 if on.as_deref() != Some(base.as_str()) {
8545 MergeOutcome {
8546 mode,
8547 ok: false,
8548 detail: format!(
8549 "{} has {} checked out, not the base branch {base}",
8550 repo.display(),
8551 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8552 ),
8553 empty: false,
8554 }
8555 } else if !git::is_clean(&repo).await? {
8556 MergeOutcome {
8557 mode,
8558 ok: false,
8559 detail: format!("{} is dirty; refusing to merge", repo.display()),
8560 empty: false,
8561 }
8562 } else {
8563 let out = match style {
8564 MergeStyle::Merge => {
8565 git::merge_no_ff(&repo, &winner.branch, &message).await?
8566 }
8567 MergeStyle::Squash => {
8568 git::merge_squash(&repo, &winner.branch, &message).await?
8569 }
8570 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8571 };
8572 MergeOutcome {
8573 mode,
8574 ok: out.ok(),
8575 detail: if out.ok() { out.stdout } else { out.stderr },
8576 empty: false,
8577 }
8578 }
8579 }
8580 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8581 self.state.pr_publication_pending = true;
8582 self.state.save()?;
8583 let out =
8584 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8585 self.state.pr_publication_pending = out.is_err();
8586 MergeOutcome {
8587 mode,
8588 ok: out.is_ok(),
8589 detail: match out {
8590 Ok(url) => url,
8591 Err(e) => format!("{e:#}"),
8592 },
8593 empty: false,
8594 }
8595 }
8596 MergeMode::Pr => {
8597 let remote = self.state.config.merge.remote.clone();
8598 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8599 if !pushed.ok() {
8600 MergeOutcome {
8601 mode,
8602 ok: false,
8603 detail: pushed.stderr,
8604 empty: false,
8605 }
8606 } else {
8607 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8613 let out = match pr_merge_plan(found) {
8614 PrPlan::Create => {
8615 gh_pr_create(
8616 &winner.worktree,
8617 &base,
8618 &winner.branch,
8619 &pr.title,
8620 &pr.body,
8621 )
8622 .await
8623 }
8624 PrPlan::Adopt { url, title } => {
8625 self.state
8626 .event("merge", format!("Pr: adopted open pull request {url}"));
8627 if self.state.github_text.as_ref().is_some_and(|g| {
8628 g.resolved
8629 && g.choice == Some(GithubTextChoice::Rewrite)
8630 && g.rewrite.is_some()
8631 }) {
8632 land::set_pr_text_verbatim(
8635 &winner.worktree,
8636 &url,
8637 &pr.title,
8638 &pr.body,
8639 )
8640 .await
8641 .map(|()| url)
8642 } else {
8643 if title != pr.title
8644 && (!is_review_run(&self.state)
8645 || leaked
8646 .as_deref()
8647 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8648 && let Err(e) =
8649 if self.state.github_text.as_ref().is_some_and(|g| {
8650 g.resolved
8651 && g.title
8652 && g.choice == Some(GithubTextChoice::Original)
8653 }) {
8654 land::set_pr_title_verbatim(
8656 &winner.worktree,
8657 &url,
8658 &pr.title,
8659 )
8660 .await
8661 } else {
8662 land::set_pr_title(
8663 &mut self.state,
8664 &winner.worktree,
8665 &url,
8666 &pr.title,
8667 )
8668 .await
8669 }
8670 {
8671 tracing::warn!("could not refresh title of {url}: {e:#}");
8672 self.state
8673 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8674 }
8675 Ok(url)
8676 }
8677 }
8678 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8679 };
8680 match out {
8681 Ok(url) => MergeOutcome {
8682 mode,
8683 ok: true,
8684 detail: url,
8685 empty: false,
8686 },
8687 Err(e) => MergeOutcome {
8688 mode,
8689 ok: false,
8690 detail: e.to_string(),
8691 empty: false,
8692 },
8693 }
8694 }
8695 }
8696 };
8697
8698 self.state.status = match (mode, outcome.ok) {
8699 (MergeMode::None, _) => RunStatus::Ready,
8700 (_, true) if outcome.empty => RunStatus::Ready,
8701 (MergeMode::Pr, true) => RunStatus::Ready,
8702 (_, true) => RunStatus::Merged,
8703 (_, false) => RunStatus::Blocked,
8704 };
8705 self.state.event(
8706 "merge",
8707 format!(
8708 "{:?}: {}",
8709 mode,
8710 outcome.detail.lines().next().unwrap_or("")
8711 ),
8712 );
8713 self.state.merge = Some(outcome);
8714 self.state.save()?;
8715
8716 if self.state.config.graph.land
8722 && mode == MergeMode::Pr
8723 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8724 {
8725 self.run_land().await?;
8726 }
8727 self.settle_questions();
8732 Ok(())
8733 }
8734
8735 async fn run_land(&mut self) -> Result<()> {
8746 crate::pr_head::ensure_owner(&self.state)?;
8747 let url = self
8748 .state
8749 .pr
8750 .as_ref()
8751 .map(|pr| pr.url.clone())
8752 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8753 .unwrap_or_default();
8754 let url = url.lines().next().unwrap_or("").trim().to_owned();
8755 if !url.starts_with("http") {
8756 return Ok(());
8757 }
8758 loop {
8761 match land::land(&mut self.state, &url).await {
8762 Ok(_)
8763 if self
8764 .state
8765 .merge_repairs
8766 .last()
8767 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8768 {
8769 if self.repair_merge_review().await? {
8770 continue;
8771 }
8772 }
8773 Ok(pr) if self.state.parked => {
8774 let _ = pr;
8778 }
8779 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8780 self.state.status = RunStatus::Superseded;
8781 self.state.save()?;
8782 }
8783 Ok(pr) => {
8784 self.state.status = match pr.state {
8785 land::PrLifecycle::Merged => RunStatus::Merged,
8786 _ => RunStatus::Blocked,
8787 };
8788 if bump::should_release_bump(self.state.status)
8795 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8796 {
8797 self.state
8803 .event("bump", format!("release bump skipped: {e:#}"));
8804 }
8805 if self.state.status == RunStatus::Merged {
8808 crate::followup::after_merge(&mut self.state, &pr.url).await;
8809 }
8810 self.state.save()?;
8811 }
8812 Err(e) => {
8813 if e.is::<crate::repo_lock::ResourceBlocked>() {
8814 return Err(e);
8815 }
8816 self.state.status = RunStatus::Blocked;
8817 self.state.event("land", format!("gave up: {e}"));
8818 self.state.save()?;
8819 }
8820 }
8821 break;
8822 }
8823 Ok(())
8824 }
8825
8826 async fn repair_merge_review(&mut self) -> Result<bool> {
8829 use crate::run::MergeRepairPhase as Phase;
8830 crate::pr_head::ensure_owner(&self.state)?;
8831 let i = self.state.merge_repairs.len() - 1;
8832 let request = self.state.merge_repairs[i].clone();
8833 if request.phase == Phase::Fixing {
8834 let restored = match self.state.winner().cloned() {
8835 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8836 None => Err(anyhow::anyhow!("repair has no winning branch")),
8837 };
8838 let why = match restored {
8839 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8840 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8841 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8842 };
8843 self.hold_merge_repair(i, why)?;
8844 return Ok(false);
8845 }
8846 let failure = if i >= self.state.config.graph.land_rounds
8847 || self.state.config.graph.review_rounds == 0
8848 {
8849 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8850 } else {
8851 None
8852 };
8853 if let Some(why) = failure {
8854 self.hold_merge_repair(i, why)?;
8855 return Ok(false);
8856 }
8857 if request.phase == Phase::Requested {
8858 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8859 Ok(pr) => pr,
8860 Err(e) => {
8861 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8862 return Ok(false);
8863 }
8864 };
8865 self.state.merge_repairs[i].phase = Phase::Fixing;
8866 self.state.land_approval = None;
8867 self.state.land_armed_head = None;
8868 self.state.save()?;
8869 let evidence = serde_json::to_string_pretty(&request)?;
8872 let reason = format!(
8873 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8874 );
8875 let budget = self.state.config.graph.land_rounds;
8876 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8877 match result {
8878 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8879 Ok(land::Fixed::Failed(why)) => {
8880 self.hold_merge_repair(
8881 i,
8882 format!(
8883 "repair failed: {why}; inspect the branch and run an explicit review"
8884 ),
8885 )?;
8886 return Ok(false);
8887 }
8888 Err(e) => {
8889 self.hold_merge_repair(
8890 i,
8891 format!(
8892 "repair failed: {e:#}; inspect the branch and run an explicit review"
8893 ),
8894 )?;
8895 return Ok(false);
8896 }
8897 }
8898 self.begin_merge_repair_review(i)?;
8902 }
8903 self.review_loop().await?;
8904 self.gate().await?;
8905 if self.resource_halt()? || self.state.parked {
8906 return Ok(false);
8907 }
8908 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8909 != Some(RunStatus::Gating)
8910 || !self.state.gate_allows_progress()
8911 {
8912 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8913 return Ok(false);
8914 }
8915 let winner = self
8918 .state
8919 .winner()
8920 .cloned()
8921 .context("repair has no winning branch")?;
8922 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8923 if self.state.gate_head.as_deref() != Some(after.as_str())
8924 || !git::is_clean(&winner.worktree).await?
8925 {
8926 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8927 return Ok(false);
8928 }
8929 let expected = self
8930 .state
8931 .pr_pushed_head
8932 .as_deref()
8933 .unwrap_or(&request.head)
8934 .to_owned();
8935 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8936 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8939 let refspec = format!("{after}:refs/heads/{public}");
8940 let push = git::git_raw(
8941 &winner.worktree,
8942 &["push", &lease, &self.state.config.merge.remote, &refspec],
8943 )
8944 .await;
8945 match push {
8946 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8947 Ok(out) => {
8948 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8949 return Ok(false);
8950 }
8951 Err(e) => {
8952 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8953 return Ok(false);
8954 }
8955 }
8956 self.state.merge_repairs[i].phase = Phase::Complete;
8957 self.state.status = RunStatus::Landing;
8958 self.state.event(
8959 "repair",
8960 "new review and gate completed; fresh owner merge decision required",
8961 );
8962 self.state.save()?;
8963 Ok(true)
8964 }
8965
8966 fn merge_repair_artifacts(&self) -> PathBuf {
8967 let artifacts = agent::artifacts_dir(&self.state.dir());
8968 match self
8969 .state
8970 .merge_repairs
8971 .last()
8972 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8973 {
8974 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8975 None => artifacts,
8976 }
8977 }
8978
8979 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8980 self.state.reviews.clear();
8981 self.state.gate.clear();
8982 self.state.gate_ran = false;
8983 self.state.gate_head = None;
8984 self.state.gate_fixes.clear();
8985 self.state.gate_repair_e2e = None;
8986 self.state.pre_gate.clear();
8987 self.state.pre_gate_commit = None;
8988 self.state.contested_handoff = None;
8989 self.state.land_reruns = None;
8990 self.state.land_approval = None;
8991 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8992 self.state.status = RunStatus::Reviewing;
8993 self.state.save()
8994 }
8995
8996 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8997 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8998 self.state.merge_repairs[index].failure = Some(why.clone());
8999 self.state.status = RunStatus::Blocked;
9000 self.state.event("repair", why);
9001 self.state.save()
9002 }
9003
9004 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9008 if let Some(existing) = self.state.seats.get(key)
9009 && existing.agent == agent
9010 {
9011 return existing.clone();
9012 }
9013 let fresh = if self.state.seats.contains_key(key) {
9016 handover_seat(key, agent, self.state.next_seat_seed())
9017 } else {
9018 SeatState::new(key, agent, self.state.seed)
9019 };
9020 self.state.seats.insert(key.to_owned(), fresh.clone());
9021 fresh
9022 }
9023
9024 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9030 match self.state.seats.get(key) {
9031 Some(s) if s.agent != spec.id => {
9032 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9033 }
9034 _ => spec,
9035 }
9036 }
9037
9038 fn view(&self, c: &Candidate) -> CandidateView {
9040 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9041 .unwrap_or_default();
9042 let (patch, _) = blind::sanitize_patch(
9043 &format!("candidate {} patch", c.label),
9044 &raw,
9045 &self.state.config.blind,
9046 );
9047 CandidateView {
9048 label: c.label,
9049 branch: c.branch.clone(),
9050 summary: c.summary.clone(),
9051 stat: c.stat.clone(),
9052 patch,
9053 }
9054 }
9055
9056 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9058 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9059 prompt::judge(
9060 "(see above)",
9061 &views,
9062 self.roles.judges.len(),
9063 base_short,
9064 "en",
9065 )
9066 }
9067
9068 fn vote_prompt_full(
9074 &self,
9075 j: usize,
9076 labels: &[char],
9077 language: &str,
9078 candidates: &[Candidate],
9079 base_short: &str,
9080 ) -> String {
9081 let mut text = format!(
9082 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9083 prompt::final_vote(labels, language),
9084 self.state.instruction,
9085 self.candidate_block(candidates, base_short)
9086 );
9087 if let Some(own) = self
9088 .state
9089 .judgements
9090 .get(j)
9091 .filter(|r| !r.ranking.is_empty())
9092 {
9093 let reasons = own
9094 .reasons
9095 .iter()
9096 .map(|(k, v)| format!("- {k}: {v}"))
9097 .collect::<Vec<_>>()
9098 .join("\n");
9099 text.push_str(&format!(
9100 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9101 own.ranking.iter().collect::<String>(),
9102 if reasons.is_empty() {
9103 ""
9104 } else {
9105 ", because:\n"
9106 }
9107 ));
9108 }
9109 if !self.state.deliberation.is_empty() {
9110 text.push_str("\n# What was argued before this vote\n");
9111 for t in self.transcript(&[], j) {
9112 text.push_str(&format!(
9113 "\n## {}{}\n\n{}\n",
9114 t.who,
9115 if t.is_self { " (you)" } else { "" },
9116 t.body.trim()
9117 ));
9118 }
9119 }
9120 text
9121 }
9122
9123 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9130 let mut turns = Vec::new();
9131 for j in &self.state.judgements {
9132 if j.ranking.is_empty() {
9133 continue;
9134 }
9135 let reasons = j
9136 .reasons
9137 .iter()
9138 .map(|(k, v)| format!("- {k}: {v}"))
9139 .collect::<Vec<_>>()
9140 .join("\n");
9141 turns.push(Turn {
9142 who: format!("Judge {} (opening ranking)", j.judge),
9143 is_self: j.judge == self_idx + 1,
9144 body: format!(
9145 "Ranked {}{}{reasons}",
9146 j.ranking.iter().collect::<String>(),
9147 if reasons.is_empty() {
9148 ""
9149 } else {
9150 ", because:\n"
9151 }
9152 ),
9153 });
9154 }
9155 for t in self
9156 .state
9157 .deliberation
9158 .iter()
9159 .flat_map(|r| r.turns.iter())
9160 .chain(current)
9161 {
9162 turns.push(Turn {
9163 who: format!("Judge {}", t.judge),
9164 is_self: t.judge == self_idx + 1,
9165 body: t.body.clone(),
9166 });
9167 }
9168 turns
9169 }
9170}
9171
9172fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9174 agent::has_session(spec.kind, seat, sessions)
9175}
9176
9177fn next_untried_in_roster<'a>(
9198 roster: &'a [AgentSpec],
9199 start: usize,
9200 tried: &BTreeSet<String>,
9201) -> Option<&'a AgentSpec> {
9202 roster
9203 .get(start + 1..)?
9204 .iter()
9205 .find(|s| !tried.contains(&s.id))
9206}
9207
9208fn pick_successor<'a>(
9229 roster: &'a [AgentSpec],
9230 start: usize,
9231 tried: &BTreeSet<String>,
9232 carried: Option<&BTreeSet<String>>,
9233 others: &BTreeSet<String>,
9234) -> Option<&'a AgentSpec> {
9235 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9236 roster
9237 .get(start + 1..)
9238 .and_then(|tail| {
9239 tail.iter()
9240 .filter(free)
9241 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9242 })
9243 .or_else(|| {
9244 carried
9245 .is_some()
9246 .then(|| roster.iter().find(free))
9247 .flatten()
9248 })
9249 .or_else(|| match carried {
9250 Some(c) => next_for_seat(roster, start, tried, c),
9251 None => next_untried_in_roster(roster, start, tried),
9252 })
9253}
9254
9255fn next_for_seat<'a>(
9265 roster: &'a [AgentSpec],
9266 start: usize,
9267 round_tried: &BTreeSet<String>,
9268 carried_failed: &BTreeSet<String>,
9269) -> Option<&'a AgentSpec> {
9270 roster
9271 .get(start + 1..)?
9272 .iter()
9273 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9274 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9275}
9276
9277fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9283 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9284 return spec;
9285 };
9286 let ok = |id: &str| !h.failed.contains(id);
9287 if let Some(last) = h.last_ok.as_deref()
9288 && ok(last)
9289 && let Some(s) = roster.iter().find(|s| s.id == last)
9290 {
9291 return s.clone();
9292 }
9293 if ok(&spec.id) {
9294 return spec;
9295 }
9296 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9297 roster
9298 .get(from + 1..)
9299 .into_iter()
9300 .flatten()
9301 .chain(roster.iter())
9302 .find(|s| ok(&s.id))
9303 .cloned()
9304 .unwrap_or(spec)
9305}
9306
9307pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9311 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9312}
9313
9314const TIMED_OUT: &str = "timed out";
9317
9318impl FailClass {
9319 fn of(out: &AgentOutcome) -> Option<Self> {
9321 match out {
9322 AgentOutcome::Ok(_) => None,
9323 AgentOutcome::Quota(_) => Some(Self::Quota),
9324 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9325 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9326 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9327 AgentOutcome::ResourceBlocked(_) => None,
9330 }
9331 }
9332
9333 fn stem_word(&self) -> &'static str {
9335 match self {
9336 Self::Quota => "quota",
9337 _ => "handover",
9338 }
9339 }
9340}
9341
9342fn failure_signature(msg: &str) -> String {
9346 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9347 if let Some(code) = line
9350 .strip_prefix("process exited with no usable output (exit code ")
9351 .and_then(|s| s.strip_suffix(')'))
9352 .and_then(|s| s.parse::<i32>().ok())
9353 {
9354 line = format!("exited with some({code}) and no usable output");
9355 } else if matches!(
9356 line.as_str(),
9357 "process exited with no usable output (no exit code reported)"
9358 | "process exited with no usable output (exit code unavailable)"
9359 ) {
9360 line = "exited with none and no usable output".to_owned();
9361 }
9362 let mut out = Vec::new();
9363 for word in line.split_whitespace() {
9364 if word.contains('/') || word.contains('\\') {
9365 out.push("<path>".to_owned());
9366 continue;
9367 }
9368 let mut w = String::new();
9369 let mut in_digits = false;
9370 for c in word.chars() {
9371 if c.is_ascii_digit() {
9372 if !in_digits {
9373 w.push('#');
9374 }
9375 in_digits = true;
9376 } else {
9377 in_digits = false;
9378 w.push(c);
9379 }
9380 }
9381 out.push(w);
9382 }
9383 out.join(" ").chars().take(120).collect()
9384}
9385
9386fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9393 match cur {
9394 FailClass::Quota | FailClass::Timeout => true,
9395 FailClass::Other(_) => prev != Some(cur),
9396 }
9397}
9398
9399fn fail_reason(out: &AgentOutcome) -> String {
9401 match out {
9402 AgentOutcome::Ok(_) => String::new(),
9403 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9404 AgentOutcome::Dropped(o) => format!(
9405 "the CLI dropped the stream ({})",
9406 o.dropped
9407 .as_ref()
9408 .map(|d| d.why.as_str())
9409 .unwrap_or("it ended without delivering its answer")
9410 ),
9411 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9412 e.lines().next().unwrap_or("").chars().take(160).collect()
9413 }
9414 }
9415}
9416
9417pub(crate) fn record_handover(
9420 state: &mut RunState,
9421 node: &str,
9422 seat: &str,
9423 from: &str,
9424 to: &str,
9425 class: &FailClass,
9426 reason: &str,
9427) {
9428 let message = if *class == FailClass::Quota {
9429 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9430 } else {
9431 format!("{seat}: handed over {from} -> {to} ({reason})")
9432 };
9433 state.event(node, message);
9434 state.handovers.push(Handover {
9435 at: Timestamp::now(),
9436 node: node.to_owned(),
9437 seat: seat.to_owned(),
9438 from: from.to_owned(),
9439 to: to.to_owned(),
9440 reason: reason.to_owned(),
9441 });
9442}
9443
9444fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9458 commands.iter().any(|c| c.exit_code.is_none())
9459}
9460
9461fn verified_noop_claim(
9474 usable: bool,
9475 commands: &[agent::CommandEvidence],
9476 text: &str,
9477) -> Option<String> {
9478 (usable && !has_unconfirmed_command(commands))
9479 .then(|| verdict::verified_noop(text))
9480 .flatten()
9481}
9482
9483fn short(commit: &str) -> String {
9484 commit.chars().take(7).collect()
9485}
9486
9487fn make_executable(path: &Path) -> Result<()> {
9488 #[cfg(unix)]
9489 {
9490 use std::os::unix::fs::PermissionsExt as _;
9491 let mut perms = std::fs::metadata(path)?.permissions();
9492 perms.set_mode(0o755);
9493 std::fs::set_permissions(path, perms)?;
9494 }
9495 #[cfg(not(unix))]
9496 {
9497 let _ = path;
9498 }
9499 Ok(())
9500}
9501
9502struct WaveCtx<'a> {
9509 run: &'a str,
9512 node: &'a str,
9514 prompts: &'a Prompts,
9515 cache: Option<&'a Path>,
9517 round: Option<usize>,
9520 carry_seats: bool,
9524 pauses: Option<(&'a Pause, &'a Pause)>,
9527}
9528
9529fn repository_wait(
9531 state: &RunState,
9532 pauses: Option<(&Pause, &Pause)>,
9533) -> crate::repo_lock::WaitPolicy {
9534 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9535 let inherited = crate::repo_lock::WaitPolicy::current();
9536 crate::repo_lock::WaitPolicy::new(
9537 Duration::from_secs(state.config.graph.timeout_resource_wait),
9538 move || {
9539 inherited.cancelled()
9540 || pauses
9541 .as_ref()
9542 .is_some_and(|(a, b)| a.parked() || b.parked())
9543 },
9544 )
9545}
9546
9547fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9548 if error.is::<crate::repo_lock::ResourceBlocked>() {
9549 state.resource_block = Some(crate::run::ResourceBlock {
9550 cause: crate::run::ResourceBlockCause::Other,
9551 node: "repository".into(),
9552 kind: if error.to_string().contains("disk pressure") {
9553 "disk pressure"
9554 } else {
9555 "repository coordination"
9556 }
9557 .into(),
9558 detail: error.to_string(),
9559 at: Timestamp::now(),
9560 });
9561 state.event("resource", error.to_string());
9562 }
9563}
9564
9565async fn coordinate_operation(
9566 state: &mut RunState,
9567 cwd: &Path,
9568 wait: &crate::repo_lock::WaitPolicy,
9569) -> Result<Option<crate::repo_lock::SeatLease>> {
9570 let result =
9571 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9572 .await;
9573 if let Err(e) = &result {
9574 record_repository_block(state, e);
9575 }
9576 result
9577}
9578
9579async fn run_one(
9581 job: SeatJob,
9582 sem: Arc<Semaphore>,
9583 ctx: &WaveCtx<'_>,
9584 state: &mut RunState,
9585 attempt: usize,
9586) -> (SeatState, AgentOutcome) {
9587 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9588 .await
9589 .pop()
9590 .expect("one job in, one result out");
9591 (seat, out)
9592}
9593
9594async fn wave(
9600 jobs: Vec<SeatJob>,
9601 sem: Arc<Semaphore>,
9602 ctx: &WaveCtx<'_>,
9603 state: &mut RunState,
9604 attempt: usize,
9605) -> Vec<(usize, SeatState, AgentOutcome)> {
9606 let WaveCtx {
9607 run,
9608 node,
9609 prompts,
9610 cache,
9611 round,
9612 carry_seats: _,
9613 pauses,
9614 } = *ctx;
9615 for job in &jobs {
9616 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9617 }
9618 if let Err(e) = state.save() {
9619 tracing::warn!("could not persist in-progress seats: {e:#}");
9624 }
9625 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9641 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9645 let cache_guard = if let Some(cache_dir) = cache {
9646 if jobs_had_a_writer {
9647 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9648 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9649 match acquire_cache_lease(
9650 state,
9651 cache_dir,
9652 &owner,
9653 resource_limit,
9654 node,
9655 node,
9656 &cancel,
9657 )
9658 .await
9659 {
9660 Ok(Some(g)) => Some(g),
9661 Ok(None) => {
9664 if let Some(block) = &mut state.resource_block {
9665 block.cause = crate::run::ResourceBlockCause::Cancelled {
9666 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9667 };
9668 }
9669 return block_wave(
9670 jobs,
9671 state,
9672 node,
9673 attempt,
9674 "parked while waiting for the shared build cache",
9675 );
9676 }
9677 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9680 Err(e) => {
9681 let why = format!("{e:#}");
9685 state.event(
9686 "resource",
9687 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9688 );
9689 return block_wave(jobs, state, node, attempt, &why);
9690 }
9691 }
9692 } else {
9693 None
9694 }
9695 } else {
9696 None
9697 };
9698 state.seats_spawning(node, attempt);
9701 if let Err(e) = state.save() {
9702 tracing::warn!("could not persist the start of a wave: {e:#}");
9703 }
9704 let slot_limit = resource_limit
9708 + jobs
9709 .iter()
9710 .map(|j| j.timeout)
9711 .max()
9712 .unwrap_or_default()
9713 .saturating_mul(jobs.len() as u32);
9714 let mut set = tokio::task::JoinSet::new();
9715 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9718 let overlay = prompts.overlay(node);
9719 let repository_wait = repository_wait(state, pauses);
9720 for (i, mut job) in jobs.into_iter().enumerate() {
9721 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9722 if cache.is_some() {
9723 job.prompt.push('\n');
9724 job.prompt
9725 .push_str(&prompt::build_cache_note(node, job.allow_write));
9726 }
9727 let sem = Arc::clone(&sem);
9728 let run = run.to_owned();
9729 let node = node.to_owned();
9730 let attachments = if node == "implement" {
9733 state.attachments.clone()
9734 } else {
9735 Vec::new()
9736 };
9737 let cache = cache
9748 .filter(|_| job.allow_write && cache_guard.is_some())
9749 .map(Path::to_path_buf);
9750 let spawned_tx = spawned_tx.clone();
9751 let repository_wait = repository_wait.clone();
9752 let coordination = state.config.coordination.clone();
9753 let fetched_base = state.fetched_base.clone();
9754 set.spawn(async move {
9755 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9759 Ok(p) => p,
9760 Err(_) => {
9761 return (
9762 i,
9763 job.seat,
9764 AgentOutcome::ResourceBlocked(format!(
9765 "no parallel slot became free within {}s",
9766 slot_limit.as_secs()
9767 )),
9768 false,
9769 );
9770 }
9771 };
9772 spawned_tx.send(job.seat.key.clone()).ok();
9773 let mut seat = job.seat;
9774 let out = repository_wait
9775 .scope(agent::invoke(
9776 &job.spec,
9777 &mut seat,
9778 &Invocation {
9779 coordination: coordination.as_ref(),
9780 fetched_base: fetched_base.as_ref(),
9781 cwd: &job.cwd,
9782 prompt: &job.prompt,
9783 timeout: job.timeout,
9784 allow_write: job.allow_write,
9785 sessions: job.sessions,
9786 artifacts: &job.artifacts,
9787 stem: &job.stem,
9788 run: &run,
9789 node: &node,
9790 cache_dir: cache.as_deref(),
9791 attachments: &attachments,
9792 writable: &[],
9793 },
9794 ))
9795 .await;
9796 let interrupted = out
9797 .as_ref()
9798 .err()
9799 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9800 let out = match out {
9801 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9802 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9803 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9811 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9812 Ok(o) => AgentOutcome::Failed(format!(
9813 "process exited with no usable output ({})",
9814 crate::notices::exit_status(o.exit_code)
9815 )),
9816 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9817 AgentOutcome::ResourceBlocked(e.to_string())
9818 }
9819 Err(e) => AgentOutcome::Failed(e.to_string()),
9820 };
9821 (i, seat, out, interrupted)
9822 });
9823 }
9824 drop(spawned_tx);
9825 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9826 loop {
9827 let joined = tokio::select! {
9828 Some(key) = spawned_rx.recv() => {
9829 state.seat_spawned(&key);
9831 if let Err(e) = state.save() {
9832 tracing::warn!("could not persist a seat's start: {e:#}");
9833 }
9834 continue;
9835 }
9836 joined = set.join_next() => joined,
9837 };
9838 let Some(joined) = joined else {
9839 break;
9840 };
9841 let (i, seat, out, interrupted) = match joined {
9842 Ok(v) => v,
9843 Err(e) => {
9847 tracing::error!("agent task panicked: {e}");
9848 continue;
9849 }
9850 };
9851 state.seat_finished(&seat.key);
9852 if interrupted
9853 && node == "implement"
9854 && let Some(label) = state
9855 .candidates
9856 .iter()
9857 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9858 .map(|candidate| candidate.label)
9859 {
9860 state.candidate_preservation.insert(
9861 label,
9862 "interrupted implementation awaiting persistence".into(),
9863 );
9864 state.interrupted_implementations.insert(label);
9865 }
9866 if let AgentOutcome::ResourceBlocked(why) = &out
9867 && state.resource_block.is_none()
9868 {
9869 state.resource_block = Some(crate::run::ResourceBlock {
9870 cause: crate::run::ResourceBlockCause::Other,
9871 node: node.to_owned(),
9872 kind: if why.contains("disk pressure") {
9873 "disk pressure"
9874 } else if why.contains("repository coordination") {
9875 "repository coordination"
9876 } else {
9877 "parallel slot"
9878 }
9879 .to_owned(),
9880 detail: why.clone(),
9881 at: Timestamp::now(),
9882 });
9883 let activity = if interrupted {
9884 "agent execution interrupted"
9885 } else {
9886 "no agent was started"
9887 };
9888 state.event("resource", format!("{node}: {activity} - {why}"));
9889 }
9890 record_jobs(state, node, round, &seat.key, &out);
9891 if let Err(e) = state.save() {
9892 tracing::warn!("could not persist a seat's completion: {e:#}");
9893 }
9894 if collected.len() <= i {
9895 collected.resize_with(i + 1, || None);
9896 }
9897 collected[i] = Some((i, seat, out));
9898 }
9899 if state
9905 .active
9906 .values()
9907 .any(|a| a.node == node && a.attempt == attempt)
9908 {
9909 state
9910 .active
9911 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9912 if let Err(e) = state.save() {
9913 tracing::warn!("could not persist the end of a wave: {e:#}");
9914 }
9915 }
9916 if let Some(cache_dir) = cache
9923 && jobs_had_a_writer
9924 {
9925 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9926 }
9927 if let Some(guard) = cache_guard {
9928 guard.release();
9929 }
9930 collected.into_iter().flatten().collect()
9931}
9932
9933fn record_jobs(
9944 state: &mut RunState,
9945 node: &str,
9946 round: Option<usize>,
9947 seat: &str,
9948 out: &AgentOutcome,
9949) {
9950 let commands: &[agent::CommandEvidence] = match out {
9951 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9952 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9953 };
9954 let checked_at = Timestamp::now();
9955 for c in commands {
9956 state.jobs.push(JobRecord {
9957 node: node.to_owned(),
9958 round,
9959 seat: seat.to_owned(),
9960 id: c.id.clone(),
9961 description: c.description.clone(),
9962 checked_at,
9963 status: match c.exit_code {
9964 Some(0) => JobStatus::Completed,
9965 Some(_) => JobStatus::Failed,
9966 None => JobStatus::Unknown,
9967 },
9968 exit_code: c.exit_code,
9969 result_summary: c.result_summary.clone(),
9970 source: c.source.clone(),
9971 });
9972 }
9973}
9974
9975fn round_is_clean(
9996 blocking: usize,
9997 e2e_ok: bool,
9998 answered: usize,
9999 expected: usize,
10000 quota_missing: usize,
10001 policy: IncompleteReviewPolicy,
10002) -> bool {
10003 if blocking != 0 || !e2e_ok {
10004 return false;
10005 }
10006 if answered == expected || policy == IncompleteReviewPolicy::Warn {
10007 return true;
10008 }
10009 answered > 0 && expected - answered <= quota_missing
10010}
10011
10012pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10036 if max_rounds == 0
10037 || reviews.last().is_some_and(|r| {
10038 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10039 })
10040 {
10041 return Some(RunStatus::Gating);
10042 }
10043 let last = reviews.last()?;
10044 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10045 if reviews.len() < max_rounds && !stagnant {
10046 return None;
10047 }
10048 if last.incomplete() && last.blocking == 0 {
10049 return Some(RunStatus::Blocked);
10050 }
10051 if last.e2e_status() == E2eStatus::ResourceBlocked
10052 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10053 {
10054 return None;
10055 }
10056 Some(if last.e2e.iter().all(CommandOutcome::ok) {
10057 RunStatus::Gating
10058 } else {
10059 RunStatus::Blocked
10060 })
10061}
10062
10063pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10078 if nudged {
10079 (full / 4).max(Duration::from_secs(120)).min(full)
10080 } else {
10081 full
10082 }
10083}
10084
10085#[allow(clippy::too_many_arguments)]
10114async fn ask_json_wave<T>(
10115 jobs: Vec<SeatJob>,
10116 sem: Arc<Semaphore>,
10117 retries: usize,
10118 roster: &[AgentSpec],
10119 ctx: &WaveCtx<'_>,
10120 losses: &mut Vec<QuotaLoss>,
10121 state: &mut RunState,
10122 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10123) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10124where
10125 T: serde::de::DeserializeOwned + Send + 'static,
10126{
10127 ask_wave_with(
10128 jobs,
10129 sem,
10130 retries,
10131 roster,
10132 ctx,
10133 losses,
10134 state,
10135 &|text: &str| {
10136 let v = verdict::extract_json::<T>(text)?;
10137 validate(&v)?;
10138 Ok(v)
10139 },
10140 )
10141 .await
10142}
10143
10144#[allow(clippy::too_many_arguments)]
10153async fn ask_wave_with<T>(
10154 jobs: Vec<SeatJob>,
10155 sem: Arc<Semaphore>,
10156 retries: usize,
10157 roster: &[AgentSpec],
10158 ctx: &WaveCtx<'_>,
10159 losses: &mut Vec<QuotaLoss>,
10160 state: &mut RunState,
10161 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10162) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10163where
10164 T: Send + 'static,
10165{
10166 let n = jobs.len();
10167 let originals: Vec<SeatJob> = jobs;
10168 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10169 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10170 let mut nudges: Vec<usize> = vec![0; n];
10177 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10181 let mut tried: Vec<BTreeSet<String>> = specs
10182 .iter()
10183 .map(|s| BTreeSet::from([s.id.clone()]))
10184 .collect();
10185 let starts: Vec<usize> = specs
10186 .iter()
10187 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10188 .collect();
10189 let carry = ctx.carry_seats && !roster.is_empty();
10190 let carried: Vec<BTreeSet<String>> = originals
10193 .iter()
10194 .map(|j| {
10195 state
10196 .seat_history
10197 .get(&j.seat.key)
10198 .filter(|_| carry)
10199 .map(|h| h.failed.clone())
10200 .unwrap_or_default()
10201 })
10202 .collect();
10203 let mut prev: Vec<Option<FailClass>> = originals
10204 .iter()
10205 .map(|j| {
10206 state
10207 .seat_history
10208 .get(&j.seat.key)
10209 .filter(|_| carry)
10210 .and_then(|h| h.last_fail.clone())
10211 })
10212 .collect();
10213 let next_agent =
10214 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10215 let others: BTreeSet<String> = specs
10216 .iter()
10217 .enumerate()
10218 .filter(|(j, _)| *j != i)
10219 .map(|(_, s)| s.id.clone())
10220 .collect();
10221 pick_successor(
10222 roster,
10223 starts[i],
10224 tried,
10225 carry.then(|| &carried[i]),
10226 &others,
10227 )
10228 .cloned()
10229 };
10230 let mut fresh: Vec<Option<String>> = vec![None; n];
10231 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10232 let mut pending: Vec<usize> = (0..n).collect();
10233
10234 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10239 for round in 0..max_rounds {
10240 if pending.is_empty() {
10241 break;
10242 }
10243 let mut batch = Vec::with_capacity(pending.len());
10244 let mut renudged: Vec<&str> = Vec::new();
10245 for &i in &pending {
10246 let src = &originals[i];
10247 let full: &str = match &src.handover {
10252 Some(h) if specs[i].id != src.spec.id => h,
10253 _ => &src.prompt,
10254 };
10255 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10258 (
10259 full.to_owned(),
10260 src.timeout,
10261 format!("{}-{word}-{}", src.stem, specs[i].id),
10262 )
10263 } else if nudges[i] == 0 {
10264 (full.to_owned(), src.timeout, src.stem.clone())
10265 } else {
10266 renudged.push(src.seat.key.as_str());
10267 let why = done[i]
10268 .as_ref()
10269 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10270 .unwrap_or_else(|| "no parsable answer".to_owned());
10271 let nudge = prompt::nudge(&why);
10272 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10273 let prompt = if nudged {
10274 nudge
10275 } else {
10276 format!("{full}\n\n---\n\n{nudge}")
10277 };
10278 (
10279 prompt,
10280 retry_budget(src.timeout, nudged),
10281 format!("{}-retry{}", src.stem, nudges[i]),
10282 )
10283 };
10284 batch.push(SeatJob {
10285 spec: specs[i].clone(),
10286 seat: seats[i].clone(),
10287 cwd: src.cwd.clone(),
10288 prompt,
10289 timeout,
10290 allow_write: src.allow_write,
10291 sessions: src.sessions,
10292 artifacts: src.artifacts.clone(),
10293 stem,
10294 handover: None,
10295 });
10296 }
10297
10298 if !renudged.is_empty() {
10299 state.event(
10300 ctx.node,
10301 format!("retry {round}: re-asking {}", renudged.join(", ")),
10302 );
10303 }
10304 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10305 let mut still = Vec::new();
10306 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10307 seats[i] = seat;
10308 if let AgentOutcome::ResourceBlocked(why) = &out {
10311 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10312 continue;
10313 }
10314 let class = FailClass::of(&out);
10315 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10318 && nudges[i] < retries
10319 && !roster.is_empty();
10320 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10321 let next = should_hand_over(prev[i].as_ref(), &cur)
10322 .then(|| next_agent(i, &tried[i], &specs))
10323 .flatten();
10324 if carry {
10325 let h = state
10326 .seat_history
10327 .entry(originals[i].seat.key.clone())
10328 .or_default();
10329 h.failed.insert(specs[i].id.clone());
10330 h.last_fail = Some(cur.clone());
10331 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10332 h.last_ok = None;
10333 }
10334 if let Err(e) = state.save() {
10337 tracing::warn!("could not persist a seat's failure history: {e:#}");
10338 }
10339 }
10340 if let Some(next) = next {
10341 record_handover(
10342 state,
10343 ctx.node,
10344 &originals[i].seat.key,
10345 &specs[i].id,
10346 &next.id,
10347 &cur,
10348 &fail_reason(&out),
10349 );
10350 tried[i].insert(next.id.clone());
10351 prev[i] = Some(cur.clone());
10352 seats[i] =
10353 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10354 specs[i] = next;
10355 fresh[i] = Some(cur.stem_word().to_owned());
10356 nudges[i] = 0;
10357 done[i] = Some(Err(anyhow::anyhow!(
10358 "handed over after: {}",
10359 fail_reason(&out)
10360 )));
10361 still.push(i);
10362 continue;
10363 }
10364 }
10365 if carry
10366 && !nudge_first
10367 && let Some(cur) = class.clone()
10368 {
10369 let h = state
10371 .seat_history
10372 .entry(originals[i].seat.key.clone())
10373 .or_default();
10374 h.failed.insert(specs[i].id.clone());
10375 h.last_fail = Some(cur);
10376 }
10377 let parsed = match out {
10378 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10379 AgentOutcome::Quota(o) => {
10380 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10381 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10382 }
10383 AgentOutcome::Dropped(o) => {
10387 let why = o
10388 .dropped
10389 .as_ref()
10390 .map(|d| d.why.as_str())
10391 .unwrap_or("the CLI ended the stream without delivering its answer");
10392 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10393 }
10394 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10395 Err(anyhow::anyhow!(e))
10396 }
10397 };
10398 let quota = class == Some(FailClass::Quota);
10399 let failed = parsed.is_err();
10400 done[i] = Some(parsed);
10401 if carry && !failed {
10402 let h = state
10403 .seat_history
10404 .entry(originals[i].seat.key.clone())
10405 .or_default();
10406 h.failed.remove(&specs[i].id);
10407 h.last_ok = Some(specs[i].id.clone());
10408 h.last_fail = None;
10409 }
10410 let agent_failure = class.is_some()
10418 && !nudge_first
10419 && !roster.is_empty()
10420 && next_agent(i, &tried[i], &specs).is_some();
10421 if failed && !quota && !agent_failure && nudges[i] < retries {
10422 nudges[i] += 1;
10423 still.push(i);
10424 }
10425 }
10426 pending = still;
10427 }
10428
10429 for (i, q) in last_quota.into_iter().enumerate() {
10432 if let Some(reset) = q {
10433 losses.push(QuotaLoss {
10434 seat: originals[i].seat.key.clone(),
10435 node: ctx.node.to_owned(),
10436 at: Timestamp::now(),
10437 reset,
10438 });
10439 }
10440 }
10441
10442 seats
10443 .into_iter()
10444 .zip(done)
10445 .zip(nudges)
10446 .map(|((seat, res), attempts)| {
10447 (
10448 seat,
10449 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10450 attempts,
10451 )
10452 })
10453 .collect()
10454}
10455
10456fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10458 results
10459 .iter()
10460 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10461}
10462
10463fn block_wave(
10467 jobs: Vec<SeatJob>,
10468 state: &mut RunState,
10469 node: &str,
10470 attempt: usize,
10471 why: &str,
10472) -> Vec<(usize, SeatState, AgentOutcome)> {
10473 state.resource_wait = None;
10474 state
10475 .active
10476 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10477 if let Err(e) = state.save() {
10478 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10479 }
10480 jobs.into_iter()
10481 .enumerate()
10482 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10483 .collect()
10484}
10485
10486async fn acquire_cache_lease(
10499 state: &mut RunState,
10500 cache_dir: &Path,
10501 owner: &crate::cache::Owner,
10502 budget: Duration,
10503 node: &str,
10504 context: &str,
10505 cancel: &(dyn Fn() -> bool + Sync),
10506) -> Result<Option<crate::cache::Guard>> {
10507 let home = crate::run::home();
10508 let started = Instant::now();
10509 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10510 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10511 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10512 Err(e) => {
10513 state.event(
10514 "verify",
10515 format!("{context}: could not check the shared build cache: {e:#}"),
10516 );
10517 if let Err(e2) = state.save() {
10518 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10519 }
10520 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10521 }
10522 };
10523 state.event(
10524 "verify",
10525 format!(
10526 "{context}: waiting for the shared build cache at {} ({})",
10527 cache_dir.display(),
10528 busy.describe()
10529 ),
10530 );
10531 state.resource_wait = Some(crate::run::ResourceWait {
10534 node: context.to_owned(),
10535 kind: "shared build cache".to_owned(),
10536 holder: busy.describe(),
10537 since: Timestamp::now(),
10538 limit_secs: budget.as_secs(),
10539 });
10540 if let Err(e) = state.save() {
10541 tracing::warn!("could not persist a cache wait: {e:#}");
10542 }
10543 let remaining = budget.saturating_sub(started.elapsed());
10544 let waited = crate::cache::wait_for_unless(
10545 &home,
10546 cache_dir,
10547 owner,
10548 remaining,
10549 Duration::from_secs(5),
10550 cancel,
10551 )
10552 .await;
10553 state.resource_wait = None;
10554 let cause = match &waited {
10555 Ok(Some(_)) => None,
10556 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10557 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10558 Some(crate::run::ResourceBlockCause::Timeout {
10559 waited_secs: started.elapsed().as_secs(),
10560 limit_secs: budget.as_secs(),
10561 })
10562 }
10563 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10564 };
10565 if let Some(cause) = cause {
10566 let detail = match &waited {
10567 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10568 Err(error) => format!("{error:#}"),
10569 };
10570 state.resource_block = Some(crate::run::ResourceBlock {
10571 cause,
10572 node: node.to_owned(),
10573 kind: "shared build cache".into(),
10574 detail,
10575 at: Timestamp::now(),
10576 });
10577 state.event(
10578 "resource",
10579 state.resource_block.as_ref().unwrap().description(),
10580 );
10581 }
10582 if let Err(error) = state.save() {
10583 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10584 }
10585 waited
10586}
10587
10588#[derive(Debug)]
10593struct CacheCheckError(String);
10594
10595impl std::fmt::Display for CacheCheckError {
10596 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10597 f.write_str(&self.0)
10598 }
10599}
10600
10601impl std::error::Error for CacheCheckError {}
10602
10603fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10605 state
10606 .resource_block
10607 .get_or_insert_with(|| crate::run::ResourceBlock {
10608 cause: crate::run::ResourceBlockCause::Other,
10609 node: node.to_owned(),
10610 kind: "shared build cache".into(),
10611 detail: format!("{error:#}"),
10612 at: Timestamp::now(),
10613 });
10614 if let Err(error) = state.save() {
10615 tracing::warn!("could not persist cache resource block: {error:#}");
10616 }
10617}
10618
10619#[allow(clippy::too_many_arguments)]
10640async fn with_cache_lease<'s, F, Fut>(
10641 state: &'s mut RunState,
10642 cache_dir: Option<&Path>,
10643 node: &str,
10644 seat: &str,
10645 worktree: &Path,
10646 head: &str,
10647 budget: Duration,
10648 context: &str,
10649 wait: &crate::repo_lock::WaitPolicy,
10650 body: F,
10651) -> Result<(Vec<CommandOutcome>, bool)>
10652where
10653 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10654 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10655{
10656 let started = Instant::now();
10659 let home = crate::run::home();
10660 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10661 let guard = if let Some(cache_dir) = cache_dir {
10662 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10663 match acquire_cache_lease(
10664 state,
10665 cache_dir,
10666 &owner,
10667 resource_limit,
10668 node,
10669 context,
10670 &|| wait.cancelled(),
10671 )
10672 .await
10673 .and_then(|g| {
10674 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10675 }) {
10676 Ok(g) => Some(g),
10677 Err(e) => {
10678 record_cache_error(state, node, &e);
10679 return Ok((
10680 vec![CommandOutcome {
10681 command: "(waiting for the shared build cache)".into(),
10682 code: None,
10683 output_tail: e.to_string(),
10684 duration_ms: started.elapsed().as_millis() as u64,
10685 resource_blocked: true,
10686 }],
10687 false,
10688 ));
10689 }
10690 }
10691 } else {
10692 None
10693 };
10694 let _coordination = coordinate_operation(state, worktree, wait).await?;
10695 if let Some(cache_dir) = cache_dir {
10696 let identity = crate::cache::Identity::new(worktree, head);
10697 if let Err(e) = crate::cache::ensure_fresh(
10698 &home,
10699 cache_dir,
10700 &identity,
10701 &state.config.verify.cargo_manifests(),
10702 ) {
10703 record_cache_error(state, node, &e);
10704 state.event(
10705 "verify",
10706 format!(
10707 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10708 worktree.display(),
10709 short(head)
10710 ),
10711 );
10712 if let Some(guard) = guard {
10713 guard.release();
10714 }
10715 return Ok((
10716 vec![CommandOutcome {
10717 command: "(confirming the shared build cache is fresh)".into(),
10718 code: None,
10719 output_tail: e.to_string(),
10720 duration_ms: started.elapsed().as_millis() as u64,
10721 resource_blocked: true,
10722 }],
10723 false,
10724 ));
10725 }
10726 }
10727 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10728 if !timed_out_pids.is_empty() {
10729 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10730 }
10731 drop(_coordination);
10734 if let Some(guard) = guard {
10735 guard.release();
10736 }
10737 Ok((outcomes, retried))
10738}
10739
10740async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10752 wait_for_pids_with(
10753 pids,
10754 crate::proc::pid_alive,
10755 LEASE_RELEASE_POLL,
10756 LEASE_RELEASE_MAX_WAIT,
10757 )
10758 .await;
10759}
10760
10761async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10767 pids: &[u32],
10768 alive: F,
10769 poll: Duration,
10770 max_wait: Duration,
10771) {
10772 let deadline = Instant::now() + max_wait;
10773 loop {
10774 if pids.iter().all(|&pid| !alive(pid)) {
10775 return;
10776 }
10777 if Instant::now() >= deadline {
10778 return;
10779 }
10780 tokio::time::sleep(poll).await;
10781 }
10782}
10783
10784fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10791 outcomes.iter().any(|o| o.resource_blocked)
10792}
10793
10794pub(crate) enum GateFix {
10796 Retry,
10798 Stop,
10801 Defer,
10804}
10805
10806fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10814 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10815 red.peek().is_some()
10816 && red.all(|o| {
10817 !o.resource_blocked
10818 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10819 && !o.output_tail.trim().is_empty()
10820 })
10821}
10822
10823fn e2e_outcome_label(o: &CommandOutcome) -> String {
10827 if o.ok() {
10828 return "pass".to_owned();
10829 }
10830 let reason = if o.build_failed() {
10831 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10832 } else {
10833 format!("FAIL ({:?})", o.code)
10834 };
10835 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10836}
10837
10838async fn run_e2e_with_retry(
10846 state: &mut RunState,
10847 shell: &[String],
10848 commands: &[String],
10849 worktree: &Path,
10850 timeout: Duration,
10851 context: &str,
10852) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10853 let (mut e2e, mut timed_out_pids) = run_commands(
10854 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10855 )
10856 .await;
10857 for o in &e2e {
10858 state.event(
10859 "verify",
10860 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10861 );
10862 }
10863 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10867 if verify_retried {
10868 state.event(
10869 "verify",
10870 format!(
10871 "{context}: verify could not build/link, not a test result — retrying once \
10872 before concluding"
10873 ),
10874 );
10875 let retried = run_commands(
10876 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10877 )
10878 .await;
10879 e2e = retried.0;
10880 timed_out_pids.extend(retried.1);
10883 for o in &e2e {
10884 state.event(
10885 "verify",
10886 format!(
10887 "{context}: retry `{}` -> {}",
10888 o.command,
10889 e2e_outcome_label(o)
10890 ),
10891 );
10892 }
10893 }
10894 (e2e, verify_retried, timed_out_pids)
10895}
10896
10897#[allow(clippy::too_many_arguments)]
10913async fn run_commands(
10914 state: &mut RunState,
10915 node: &str,
10916 task: &str,
10917 attempt: usize,
10918 shell: &[String],
10919 commands: &[String],
10920 cwd: &Path,
10921 timeout: Duration,
10922) -> (Vec<CommandOutcome>, Vec<u32>) {
10923 if commands.is_empty() {
10924 return (Vec::new(), Vec::new());
10929 }
10930 let mut out = Vec::new();
10931 let mut timed_out_pids = Vec::new();
10932 let total = commands.len();
10933 for (idx, command) in commands.iter().enumerate() {
10934 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10935 if let Err(e) = state.save() {
10936 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10937 }
10938 let started = Instant::now();
10939 let mut cmd = tokio::process::Command::new(&shell[0]);
10940 cmd.quiet();
10941 for key in crate::worker::INVOCATION_ENV {
10945 cmd.env_remove(key);
10946 }
10947 cmd.args(&shell[1..])
10948 .arg(command)
10949 .current_dir(cwd)
10950 .stdin(std::process::Stdio::null())
10951 .stdout(std::process::Stdio::piped())
10952 .stderr(std::process::Stdio::piped())
10953 .kill_on_drop(true);
10954 let mut outputs = Vec::new();
10955 let protected = (|| -> Result<()> {
10956 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10957 outputs.push(guard);
10958 }
10959 if let Some(cache) = state.config.cache_dir() {
10960 outputs.push(crate::retention::OutputLease::cache(
10961 &crate::run::home(),
10962 &cache,
10963 false,
10964 )?);
10965 }
10966 Ok(())
10967 })();
10968 if let Err(e) = protected {
10969 out.push(CommandOutcome {
10970 command: command.clone(),
10971 code: None,
10972 output_tail: format!("output ownership unavailable: {e:#}"),
10973 duration_ms: 0,
10974 resource_blocked: true,
10975 });
10976 continue;
10977 }
10978 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10979 let (code, body) = match spawned {
10980 Ok(mut running) => {
10981 let pid = running.child().id();
10982 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10983 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10984 let mut paths = vec![cwd.to_owned()];
10985 if let Some(cache) = state.config.cache_dir() {
10986 paths.push(cache);
10987 }
10988 let mut pressure = None;
10989 let status = tokio::select! {
10990 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10991 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10992 pressure = Some(reason); None
10993 }
10994 };
10995 let (code, failure) = match status {
10996 Some(Ok(Ok(status))) => (
10997 status.code(),
10998 status
10999 .code()
11000 .is_none()
11001 .then(|| format!("command terminated: {status}")),
11002 ),
11003 Some(Ok(Err(error))) => {
11004 (None, Some(format!("failed waiting for command: {error}")))
11005 }
11006 Some(Err(_)) => {
11007 if !running.coordinated()
11011 && let Some(pid) = pid
11012 {
11013 timed_out_pids.push(pid);
11014 }
11015 (
11016 None,
11017 Some(format!("timed out after {}s", timeout.as_secs())),
11018 )
11019 }
11020 None => (None, None),
11021 };
11022 running.finish().await;
11023 if let Some(reason) = pressure {
11024 state.resource_block = Some(crate::run::ResourceBlock {
11025 cause: crate::run::ResourceBlockCause::Other,
11026 node: node.to_owned(),
11027 kind: "disk pressure".to_owned(),
11028 detail: reason.clone(),
11029 at: Timestamp::now(),
11030 });
11031 out.push(CommandOutcome {
11032 command: command.clone(),
11033 code: None,
11034 output_tail: reason,
11035 duration_ms: started.elapsed().as_millis() as u64,
11036 resource_blocked: true,
11037 });
11038 break;
11039 }
11040 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11041 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11042 if let Some(failure) = failure {
11043 body.push('\n');
11044 body.push_str(&failure);
11045 }
11046 (code, body)
11047 }
11048 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11049 };
11050 out.push(CommandOutcome {
11051 command: command.clone(),
11052 code,
11053 output_tail: tail(&body, OUTPUT_TAIL),
11054 duration_ms: started.elapsed().as_millis() as u64,
11055 resource_blocked: false,
11056 });
11057 }
11058 state.task_finished(task);
11059 if let Err(e) = state.save() {
11060 tracing::warn!("could not persist the end of {task}: {e:#}");
11061 }
11062 (out, timed_out_pids)
11063}
11064
11065fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11077 let repo = repo.display();
11078 match style {
11079 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11080 MergeStyle::Squash => {
11081 let subject = message
11084 .lines()
11085 .next()
11086 .unwrap_or(branch)
11087 .replace(['\\', '"', '$', '`'], "");
11088 format!(
11089 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11090 )
11091 }
11092 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11093 }
11094}
11095
11096const PR_TITLE_MAX: usize = 240;
11107
11108fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11119 let line = queue::first_line(text)?;
11120 let chars: Vec<char> = line.chars().collect();
11121 let (kept, mark) = if chars.len() <= max {
11122 (line.to_owned(), "")
11123 } else if let Some(end) = sentence_end(&chars, max) {
11124 (chars[..end].iter().collect::<String>(), "")
11125 } else {
11126 let room = max.saturating_sub(3);
11127 let cut = if chars[room].is_whitespace() {
11130 room
11131 } else {
11132 chars[..room]
11133 .iter()
11134 .rposition(|c| c.is_whitespace())
11135 .unwrap_or(room)
11136 };
11137 let head: String = chars[..cut].iter().collect();
11138 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11139 (head.to_owned(), "...")
11140 };
11141 if kept.is_empty() {
11142 return None;
11143 }
11144 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11145 return None;
11146 }
11147 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11148 return None;
11149 }
11150 Some(format!("{kept}{mark}"))
11151}
11152
11153fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11157 const MIN: usize = 20;
11158 for i in MIN..max.min(chars.len()) {
11159 if !matches!(chars[i], '.' | '!' | '?') {
11160 continue;
11161 }
11162 let Some(&next) = chars.get(i + 1) else {
11163 continue;
11164 };
11165 if !next.is_whitespace() {
11166 continue;
11167 }
11168 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11169 if after.is_some_and(|c| c.is_ascii_lowercase()) {
11170 continue;
11171 }
11172 let word: String = chars[..i]
11173 .iter()
11174 .rev()
11175 .take_while(|c| !c.is_whitespace())
11176 .collect::<Vec<_>>()
11177 .into_iter()
11178 .rev()
11179 .collect();
11180 let word = word.to_ascii_lowercase();
11181 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11182 continue;
11183 }
11184 let end = chars[..i]
11185 .iter()
11186 .rposition(|c| !c.is_whitespace())
11187 .map_or(i, |p| p + 1);
11188 return Some(end);
11189 }
11190 None
11191}
11192
11193struct PrMessage {
11198 title: String,
11199 body: String,
11200}
11201
11202impl PrMessage {
11203 fn commit_message(&self) -> String {
11207 format!("{}\n\n{}", self.title, self.body)
11208 }
11209}
11210
11211fn title_marker(line: &str) -> Option<&str> {
11213 let line = line.trim();
11214 let head = line.get(..6)?;
11215 head.eq_ignore_ascii_case("title:")
11216 .then(|| line[6..].trim())
11217}
11218
11219fn summary_title(summary: &str) -> Option<String> {
11224 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11225 let raw = title_marker(first)?;
11226 if raw.is_empty() {
11227 return None;
11228 }
11229 let title = queue::title_from(raw, PR_TITLE_MAX);
11230 let lower = title.to_ascii_lowercase();
11231 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11232 return None;
11233 }
11234 Some(title)
11235}
11236
11237const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11239
11240pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11242
11243pub(crate) fn is_review_run(state: &RunState) -> bool {
11247 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11248}
11249
11250fn review_title(state: &RunState) -> Option<String> {
11255 english_subject(
11256 state.reviewed_commits.as_ref()?.first()?,
11257 state.config.graph.github_english(),
11258 )
11259}
11260
11261fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11264 let raw = raw.trim();
11265 if raw.is_empty() {
11266 return None;
11267 }
11268 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11269 return None;
11270 }
11271 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11272 return None;
11273 }
11274 let title = queue::title_from(raw, PR_TITLE_MAX);
11275 let lower = title.to_ascii_lowercase();
11276 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11277 return None;
11278 }
11279 Some(title)
11280}
11281
11282#[derive(Debug, Clone, PartialEq, Eq)]
11285struct BranchFacts {
11286 commits: Vec<(String, String)>,
11288 stat: String,
11290}
11291
11292const STAT_FILE_LINES: usize = 25;
11294const COMMITS_MAX_CHARS: usize = 20_000;
11296
11297async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11300 let commits = git::commit_log(repo, base, branch).await.ok()?;
11301 if commits.is_empty() {
11302 return None;
11303 }
11304 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11305 let lines: Vec<&str> = stat.lines().collect();
11306 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11307 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11308 let more = format!(" ... {omitted} more file(s)");
11309 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11310 kept.push(&more);
11311 kept.push(lines[lines.len() - 1]);
11312 kept.join("\n")
11313 } else {
11314 lines.join("\n")
11315 };
11316 Some(BranchFacts { commits, stat })
11317}
11318
11319fn markdown_safe(text: &str) -> String {
11322 text.replace("</details>", "</details>")
11323 .replace("\x60\x60\x60", "~~~")
11324}
11325
11326fn neutral_title(state: &RunState, winner: char) -> String {
11327 format!(
11328 "chore: land candidate {} of run {}",
11329 winner.to_ascii_uppercase(),
11330 state.id
11331 )
11332}
11333
11334pub(crate) async fn repair_landing_gate(
11337 state: &mut RunState,
11338 winner: &Candidate,
11339 outcomes: &[CommandOutcome],
11340) -> Result<GateFix> {
11341 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11342 return Ok(GateFix::Stop);
11343 }
11344 let mut runner = Runner {
11345 roles: state.config.resolve_roles()?,
11346 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11347 state: state.clone(),
11348 pause: Pause::new(),
11349 interrupt: Pause::new(),
11350 };
11351 let result = runner.gate_fix_round(winner, outcomes).await;
11352 *state = runner.state;
11353 result
11354}
11355
11356pub(crate) async fn check_gate_repair_e2e(
11359 state: &mut RunState,
11360 winner: &Candidate,
11361 wait: &crate::repo_lock::WaitPolicy,
11362) -> Result<GateFix> {
11363 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11364 return Ok(GateFix::Retry);
11365 }
11366 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11367 if state
11368 .gate_repair_e2e
11369 .as_ref()
11370 .is_some_and(|r| r.head == head && r.passed)
11371 {
11372 return Ok(GateFix::Retry);
11373 }
11374 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11375 head: head.clone(),
11376 passed: false,
11377 outcomes: Vec::new(),
11378 });
11379 state.save()?;
11380 let commands = state.config.verify.e2e.clone();
11381 let outcomes = if commands.is_empty() {
11382 Vec::new()
11383 } else {
11384 let shell = state.config.shell();
11385 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11386 let cache_dir = state.config.cache_dir();
11387 let (outcomes, _) = with_cache_lease(
11388 state,
11389 cache_dir.as_deref(),
11390 "e2e",
11391 "e2e",
11392 &winner.worktree,
11393 &head,
11394 timeout,
11395 "gate repair e2e",
11396 wait,
11397 |state, budget| async move {
11398 run_e2e_with_retry(
11399 state,
11400 &shell,
11401 &commands,
11402 &winner.worktree,
11403 budget,
11404 "gate repair e2e",
11405 )
11406 .await
11407 },
11408 )
11409 .await?;
11410 outcomes
11411 };
11412 let deferred = verify_inconclusive(&outcomes);
11413 let passed = outcomes.iter().all(CommandOutcome::ok)
11414 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11415 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11416 head,
11417 passed,
11418 outcomes,
11419 });
11420 state.save()?;
11421 if deferred {
11422 Ok(GateFix::Defer)
11423 } else if !passed {
11424 state.event(
11425 "gate",
11426 "verify.e2e failed after gate repair; nothing published",
11427 );
11428 Ok(GateFix::Stop)
11429 } else {
11430 Ok(GateFix::Retry)
11431 }
11432}
11433
11434pub(crate) async fn run_landing_gate(
11436 state: &mut RunState,
11437 winner: &Candidate,
11438 wait: &crate::repo_lock::WaitPolicy,
11439) -> Result<Vec<CommandOutcome>> {
11440 let shell = state.config.shell();
11441 let gate_commands = state.config.verify.gate.clone();
11442 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11443 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11444 Vec::new()
11445 } else {
11446 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11447 let cache_dir = state.config.cache_dir();
11448 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11449 let (outcomes, _) = with_cache_lease(
11450 state,
11451 cache_dir.as_deref(),
11452 "gate",
11453 "gate",
11454 &winner.worktree,
11455 &head,
11456 timeout,
11457 "final gate",
11458 wait,
11459 |state, budget| {
11460 let shell = shell.clone();
11461 let gate_commands = gate_commands.clone();
11462 let winner = winner.clone();
11463 async move {
11464 Runner::run_pre_gate(state, &winner).await;
11465 let (outcomes, timed_out_pids) = run_commands(
11466 state,
11467 "gate",
11468 "gate",
11469 0,
11470 &shell,
11471 &gate_commands,
11472 &winner.worktree,
11473 budget,
11474 )
11475 .await;
11476 (outcomes, false, timed_out_pids)
11477 }
11478 },
11479 )
11480 .await?;
11481 outcomes
11482 };
11483 if outcomes.is_empty() {
11484 state.event("gate", "no gate commands configured; not tested");
11489 }
11490 for o in &outcomes {
11491 state.event(
11492 "gate",
11493 format!(
11494 "`{}` -> {}",
11495 o.command,
11496 if o.ok() {
11497 "pass".to_owned()
11498 } else {
11499 format!(
11500 "FAIL ({:?})\n{}",
11501 o.code,
11502 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11503 )
11504 }
11505 ),
11506 );
11507 }
11508 Ok(outcomes)
11509}
11510
11511pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11517 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11518 ""
11519 } else {
11520 pr_title
11521 }
11522}
11523
11524pub fn landing_subject_source(state: &RunState) -> String {
11528 if is_review_run(state) {
11529 let winner = state.candidates.first().map_or('A', |c| c.label);
11530 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11531 }
11532 state.instruction.clone()
11533}
11534
11535fn summary_without_title(summary: &str) -> String {
11538 let mut lines = summary.trim().lines().peekable();
11539 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11540 lines.next();
11541 }
11542 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11543}
11544
11545#[cfg(test)]
11557fn pr_message(state: &RunState, winner: char) -> PrMessage {
11558 pr_message_with(state, winner, None)
11559}
11560
11561#[cfg(test)]
11564fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11565 let raw = pr_message_raw(
11566 state,
11567 winner,
11568 facts,
11569 crate::pr_outcome::FinalHead::Unreported,
11570 ChangeNote::NotAsked,
11571 );
11572 let id = crate::scrub::Identity::current();
11573 PrMessage {
11574 title: crate::scrub::scrub(&raw.title, &id),
11575 body: crate::scrub::scrub(&raw.body, &id),
11576 }
11577}
11578
11579#[derive(Debug, Clone, Copy)]
11581enum ChangeNote<'a> {
11582 NotAsked,
11584 Summary(&'a crate::run::ChangeSummary),
11586 Unavailable(&'a str),
11588}
11589
11590fn pr_message_raw(
11591 state: &RunState,
11592 winner: char,
11593 facts: Option<&BranchFacts>,
11594 head: crate::pr_outcome::FinalHead<'_>,
11595 change: ChangeNote<'_>,
11596) -> PrMessage {
11597 if !matches!(change, ChangeNote::NotAsked) {
11598 let (title, mut body) = match change {
11599 ChangeNote::Summary(c) => (
11600 c.title.clone(),
11601 format!("## Summary\n\n{}\n\n", c.body.trim()),
11602 ),
11603 ChangeNote::Unavailable(why) => (
11604 neutral_title(state, winner),
11605 format!(
11606 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11607 ),
11608 ),
11609 ChangeNote::NotAsked => unreachable!(),
11610 };
11611 body.push_str(&public_pr_evidence(state, head));
11612 body.push_str(&format!(
11613 "\n---\nmagi:run/{} magi:candidate-{}\n",
11614 state.id,
11615 winner.to_ascii_lowercase()
11616 ));
11617 return PrMessage { title, body };
11618 }
11619 pr_preview(state, winner, facts, head)
11620}
11621
11622fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11625 let mut body = crate::pr_outcome::review_rounds(state);
11626 body.push_str(&crate::pr_outcome::final_verification(state, head));
11627 let open = state.last_round_findings();
11628 if !open.is_empty() {
11629 body.push_str("## Open review findings\n\n");
11630 for f in open {
11631 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11632 }
11633 body.push('\n');
11634 }
11635 if let Some(c) = state.change_summary.as_ref()
11636 && head == crate::pr_outcome::FinalHead::At(&c.head)
11637 {
11638 body.push_str(&format!(
11639 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11640 c.base, c.head
11641 ));
11642 }
11643 body
11644}
11645
11646fn pr_preview(
11648 state: &RunState,
11649 winner: char,
11650 facts: Option<&BranchFacts>,
11651 head: crate::pr_outcome::FinalHead<'_>,
11652) -> PrMessage {
11653 let summary = state
11654 .candidates
11655 .iter()
11656 .find(|c| c.label == winner)
11657 .map(|c| c.summary.as_str())
11658 .unwrap_or_default();
11659 let review = is_review_run(state);
11662 let title = if review {
11663 facts
11664 .and_then(|f| {
11665 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11666 })
11667 .or_else(|| review_title(state))
11668 .or_else(|| {
11669 state
11670 .candidates
11671 .iter()
11672 .find(|c| c.label == winner)
11673 .filter(|c| !c.branch.starts_with("magi/"))
11674 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11675 })
11676 .unwrap_or_else(|| neutral_title(state, winner))
11677 } else {
11678 summary_title(summary).unwrap_or_else(|| {
11679 english_title(
11680 &state.instruction,
11681 PR_TITLE_MAX,
11682 state.config.graph.github_english(),
11683 )
11684 .unwrap_or_else(|| neutral_title(state, winner))
11685 })
11686 };
11687
11688 let mut body = String::new();
11689 let what = summary_without_title(summary);
11690 if !what.is_empty() {
11691 body.push_str("## Summary\n\n");
11692 body.push_str(&what);
11693 body.push_str("\n\n");
11694 }
11695
11696 body.push_str(&crate::pr_outcome::review_rounds(state));
11698 body.push_str(&crate::pr_outcome::final_verification(state, head));
11699 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11700
11701 let open = state.open_findings();
11702 if !open.is_empty() {
11703 body.push_str("## Open review findings\n\n");
11704 for f in &open {
11705 body.push_str(&format!(
11706 "- `{}` [{:?}] {}\n",
11707 f.id, f.severity, "Context retained in local review records."
11708 ));
11709 }
11710 body.push('\n');
11711 }
11712
11713 if let Some(fix) = fix
11714 && !fix.rejected.is_empty()
11715 {
11716 body.push_str("## Declined by the fixer\n\n");
11717 for r in &fix.rejected {
11718 body.push_str(&format!(
11719 "- `{}`: {}\n",
11720 r.id, "Reason retained in local review records."
11721 ));
11722 }
11723 body.push('\n');
11724 }
11725
11726 if review {
11727 body.push_str("## Commits under review\n\n");
11729 if let Some(facts) = facts {
11730 let mut left = COMMITS_MAX_CHARS;
11731 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11732 let mut entry = format!("- {}\n", markdown_safe(subject));
11733 for l in markdown_safe(text).lines() {
11734 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11735 }
11736 if left == 0 {
11737 body.push_str(&format!(
11738 "- ... {} more commit(s)\n",
11739 facts.commits.len() - i
11740 ));
11741 break;
11742 }
11743 if entry.len() > left {
11744 let mut end = left;
11747 while !entry.is_char_boundary(end) {
11748 end -= 1;
11749 }
11750 entry.truncate(end);
11751 entry.push_str("\n ... (truncated)\n");
11752 left = 0;
11753 } else {
11754 left -= entry.len();
11755 }
11756 body.push_str(&entry);
11757 }
11758 if !facts.stat.trim().is_empty() {
11759 body.push_str(&format!(
11760 "\n## Diff stat\n\n```\n{}\n```\n",
11761 markdown_safe(facts.stat.trim())
11762 ));
11763 }
11764 } else {
11765 match &state.reviewed_commits {
11766 Some(subjects) => {
11767 for s in subjects {
11768 body.push_str(&format!("- {}\n", s.trim()));
11769 }
11770 }
11771 None => {
11772 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11775 body.push_str(rest.trim());
11776 body.push('\n');
11777 }
11778 }
11779 }
11780 } else {
11781 let task = state.instruction.trim();
11782 let task = if task.is_empty() {
11783 "(empty task)"
11784 } else {
11785 task
11786 };
11787 body.push_str(&format!(
11788 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11789 task.replace("</details>", "</details>")
11790 ));
11791 }
11792
11793 body.push_str(&format!(
11794 "\n---\nmagi:run/{} magi:candidate-{}\n",
11795 state.id,
11796 winner.to_ascii_lowercase()
11797 ));
11798
11799 PrMessage { title, body }
11800}
11801
11802fn seeded_instruction(state: &RunState) -> String {
11805 let mut inputs = Vec::new();
11806 if let Some(start) = &state.sources.start_from {
11807 inputs.push(format!(
11808 "Start from `{}` at `{}` (including ancestry).",
11809 start.token, start.sha
11810 ));
11811 }
11812 for pick in &state.sources.pick_commits {
11813 inputs.push(format!(
11814 "Pick `{}` at `{}` (single commit).",
11815 pick.token, pick.sha
11816 ));
11817 }
11818 if inputs.is_empty() {
11819 state.instruction.clone()
11820 } else {
11821 format!(
11822 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11823 state.instruction,
11824 inputs.join("\n")
11825 )
11826 }
11827}
11828
11829async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11833 let base = &state.base_branch;
11834 let mut against = base.clone();
11835 if mode == MergeMode::Pr {
11836 let remote = &state.config.merge.remote;
11840 let tracking = format!("{remote}/{base}");
11841 let fetched = git::fetch(repo, remote, base).await;
11842 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11843 against = tracking;
11844 } else {
11845 return false;
11846 }
11847 }
11848 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11849}
11850
11851fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11854 let mut detail = format!(
11855 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11856 nothing to open a pull request for"
11857 );
11858 detail.push_str(
11859 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11860 );
11861 detail
11862}
11863
11864#[derive(Debug, PartialEq, Eq)]
11867enum PrPlan {
11868 Create,
11869 Adopt { url: String, title: String },
11870 Stop(String),
11871}
11872
11873fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11876 match found {
11877 Ok(land::OpenPr::None) => PrPlan::Create,
11878 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11879 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11880 "several open pull requests exist for this branch, not picking one: {}",
11881 urls.join(" ")
11882 )),
11883 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11884 }
11885}
11886
11887pub(crate) async fn gh_pr_create(
11889 cwd: &Path,
11890 base: &str,
11891 head: &str,
11892 title: &str,
11893 body: &str,
11894) -> Result<String> {
11895 let out = tokio::process::Command::new("gh")
11896 .args([
11897 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11898 ])
11899 .current_dir(cwd)
11900 .quiet()
11901 .stdin(std::process::Stdio::null())
11902 .output()
11903 .await
11904 .context("spawn gh")?;
11905 if out.status.success() {
11906 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11907 } else {
11908 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11909 }
11910}
11911
11912async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11914 match std::fs::symlink_metadata(path) {
11915 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11916 git::worktree_remove_clean(repo, path).await?;
11919 return Ok(());
11920 }
11921 Err(e) => return Err(e.into()),
11922 Ok(_) => {}
11923 }
11924 if !git::is_clean(path).await? {
11925 bail!(
11926 "worktree has uncommitted work; refusing removal: {}",
11927 path.display()
11928 );
11929 }
11930 if !git::worktree_remove_clean(repo, path).await? {
11931 bail!("worktree removal refused: {}", path.display());
11932 }
11933 Ok(())
11934}
11935
11936pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11945 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11946 bail!("refusing fold: {reason}");
11947 }
11948 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11949 bail!("candidate preservation incomplete; refusing cleanup");
11950 }
11951 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11955 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11956 let _driver = crate::driver::maintenance(home, &state.id)?;
11957 let mut output_guards = Vec::new();
11958 for entry in std::fs::read_dir(state.worktree_root())
11959 .into_iter()
11960 .flatten()
11961 .flatten()
11962 {
11963 if entry.path().join(".git").is_file() {
11964 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11965 }
11966 }
11967 let repo = state.repo.clone();
11968 let wait = repository_wait(state, None);
11971 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11972 Ok(lease) => lease,
11973 Err(e) => {
11974 state.save_under(home)?;
11975 return Err(e);
11976 }
11977 };
11978 let _source_outputs = if state.status == RunStatus::Merged
11981 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11982 {
11983 verify_merged_fold_sources(state, drop_winner, home).await?
11984 } else {
11985 Vec::new()
11986 };
11987 let root = state.worktree_root();
11988 let winner = state.tally.as_ref().map(|t| t.winner);
11989 let mut removed = Vec::new();
11990
11991 for i in 0..state.candidates.len() {
11992 let c = state.candidates[i].clone();
11993 let is_winner = Some(c.label) == winner;
11994 if is_winner && !drop_winner {
11995 continue;
11996 }
11997 let existed = c.worktree.exists();
11998 remove_preserved_worktree(&repo, &c.worktree).await?;
12001 if existed {
12002 removed.push(c.worktree.to_string_lossy().into_owned());
12003 }
12004 let handed_over = state.released_branches.contains(&c.branch);
12007 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12008 if !git::branch_delete(&repo, &c.branch).await? {
12009 bail!("candidate branch removal refused");
12010 }
12011 removed.push(c.branch.clone());
12012 }
12013 state.candidates[i].folded = true;
12014 }
12015
12016 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12017 let path = name.path();
12018 let keep = !drop_winner
12019 && winner.is_some_and(|w| {
12020 path.file_name()
12021 .is_some_and(|n| n == format!("cand-{w}").as_str())
12022 });
12023 if keep {
12024 continue;
12025 }
12026 remove_preserved_worktree(&repo, &path).await?;
12027 removed.push(path.to_string_lossy().into_owned());
12028 }
12029
12030 remove_if_empty(&root);
12039
12040 if state.enabled_worktree_config && drop_winner {
12041 git::release_worktree_config(&repo).await.ok();
12045 state.enabled_worktree_config = false;
12046 }
12047 state.save_under(home)?;
12048 Ok(removed)
12049}
12050
12051async fn verify_merged_fold_sources(
12052 state: &RunState,
12053 drop_winner: bool,
12054 home: &Path,
12055) -> Result<Vec<std::fs::File>> {
12056 let winner = state.tally.as_ref().map(|t| t.winner);
12057 let mut heads = BTreeSet::new();
12058 for candidate in &state.candidates {
12059 if Some(candidate.label) == winner && !drop_winner {
12060 continue;
12061 }
12062 anyhow::ensure!(
12063 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12064 "candidate recovery source is outside the inventoried run root: {}",
12065 candidate.worktree.display()
12066 );
12067 match std::fs::symlink_metadata(&candidate.worktree) {
12068 Ok(_) => {}
12069 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12070 Err(error) => return Err(error.into()),
12071 }
12072 if !state.released_branches.contains(&candidate.branch)
12073 && git::branch_exists(&state.repo, &candidate.branch).await?
12074 {
12075 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12076 }
12077 }
12078 let mut cargo_guards = Vec::new();
12079 let entries = match std::fs::read_dir(state.worktree_root()) {
12080 Ok(entries) => Some(entries),
12081 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12082 Err(error) => return Err(error.into()),
12083 };
12084 for entry in entries.into_iter().flatten() {
12085 let path = entry?.path();
12086 let metadata = std::fs::symlink_metadata(&path)?;
12087 anyhow::ensure!(
12088 metadata.is_dir() && !metadata.file_type().is_symlink(),
12089 "non-directory or symlink recovery source retained: {}",
12090 path.display()
12091 );
12092 if !drop_winner
12093 && winner.is_some_and(|w| {
12094 path.file_name()
12095 .is_some_and(|name| name == format!("cand-{w}").as_str())
12096 })
12097 {
12098 continue;
12099 }
12100 anyhow::ensure!(
12101 path.join(".git").is_file(),
12102 "unregistered recovery source retained: {}",
12103 path.display()
12104 );
12105 anyhow::ensure!(
12106 git::git(
12107 &path,
12108 &[
12109 "status",
12110 "--porcelain",
12111 "--untracked-files=all",
12112 "--ignored",
12113 "--",
12114 ".",
12115 ":(exclude)target"
12116 ]
12117 )
12118 .await?
12119 .is_empty(),
12120 "source changes or ignored recovery files retained: {}",
12121 path.display()
12122 );
12123 match std::fs::symlink_metadata(path.join("target")) {
12124 Ok(_) => {
12125 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12126 }
12127 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12128 Err(error) => return Err(error.into()),
12129 }
12130 for checkpoint in [
12131 "rebase-merge",
12132 "rebase-apply",
12133 "sequencer",
12134 "CHERRY_PICK_HEAD",
12135 "MERGE_HEAD",
12136 "REBASE_HEAD",
12137 "REVERT_HEAD",
12138 "BISECT_LOG",
12139 "AUTO_MERGE",
12140 "MERGE_MSG",
12141 ] {
12142 let location = git::git(
12143 &path,
12144 &[
12145 "rev-parse",
12146 "--path-format=absolute",
12147 "--git-path",
12148 checkpoint,
12149 ],
12150 )
12151 .await?;
12152 anyhow::ensure!(
12153 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12154 if e.kind() == std::io::ErrorKind::NotFound),
12155 "Git recovery checkpoint retained: {checkpoint}"
12156 );
12157 }
12158 heads.insert(git::rev_parse(&path, "HEAD").await?);
12159 }
12160 crate::fold_evidence::preserve(state, home, &heads).await?;
12161
12162 Ok(cargo_guards)
12163}
12164
12165fn remove_if_empty(dir: &Path) {
12176 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12177 std::fs::remove_dir(dir).ok();
12178 }
12179}
12180
12181pub fn worst_open(state: &RunState) -> Option<Severity> {
12183 state
12184 .reviews
12185 .last()?
12186 .reviews
12187 .iter()
12188 .flat_map(|r| r.findings.iter())
12189 .map(|f| f.severity)
12190 .max()
12191}
12192
12193async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12197 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12198 git::acquire_worktree_config(repo).await?;
12199 state.enabled_worktree_config = true;
12200 }
12201 Ok(())
12202}
12203
12204#[cfg(test)]
12205mod tests {
12206 #[tokio::test]
12207 async fn launcher_base_fetch_requires_remote_success() {
12208 let tmp = tempfile::tempdir().unwrap();
12209 let repo = tmp.path().join("repo");
12210 let remote = tmp.path().join("remote");
12211 std::fs::create_dir(&repo).unwrap();
12212 std::fs::create_dir(&remote).unwrap();
12213 git::git(&remote, &["init", "--bare", "-b", "main"])
12214 .await
12215 .unwrap();
12216 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12217 git::git(
12218 &repo,
12219 &[
12220 "-c",
12221 "user.name=Fixture",
12222 "-c",
12223 "user.email=fixture@example.invalid",
12224 "commit",
12225 "--allow-empty",
12226 "-m",
12227 "fixture",
12228 ],
12229 )
12230 .await
12231 .unwrap();
12232 git::git(
12233 &repo,
12234 &["remote", "add", "origin", remote.to_str().unwrap()],
12235 )
12236 .await
12237 .unwrap();
12238 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12239 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12240 assert_eq!(
12241 resolve_base(&repo, "main", "origin", None)
12242 .await
12243 .unwrap()
12244 .commit,
12245 head
12246 );
12247 #[cfg(unix)]
12248 {
12249 let policy: crate::config::Coordination = toml::from_str(
12250 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12251 ).unwrap();
12252 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12253 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12254 let held = crate::repo_lock::SeatLease::operation_with(
12255 &repo,
12256 Some(&policy),
12257 &crate::repo_lock::WaitPolicy::default(),
12258 )
12259 .await
12260 .unwrap();
12261 assert!(held.is_some());
12262 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12263 tokio::pin!(fetch);
12264 assert!(
12265 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12266 .await
12267 .is_err()
12268 );
12269 assert!(
12270 !repo.join(".git/FETCH_HEAD").exists(),
12271 "no fetch before admission"
12272 );
12273 drop(held);
12274 let fetched = fetch.await.unwrap();
12275 assert_eq!(fetched.commit, head);
12276 assert_eq!(fetched.remote, "origin");
12277 assert_eq!(fetched.branch, "main");
12278 }
12279 git::git(
12281 &repo,
12282 &[
12283 "remote",
12284 "set-url",
12285 "origin",
12286 tmp.path().join("missing").to_str().unwrap(),
12287 ],
12288 )
12289 .await
12290 .unwrap();
12291 let error = resolve_base(&repo, "main", "origin", None)
12292 .await
12293 .unwrap_err()
12294 .to_string();
12295 assert!(error.contains("cannot read origin/main"));
12296 assert!(error.contains("refusing to branch off"));
12297 }
12298
12299 #[test]
12300 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12301 let leaked = vec!["chore(deps): update a crate".to_owned()];
12302 let own = "fix(daemon): apply a chosen action";
12303 assert!(should_retitle("", own, &leaked));
12304 assert!(should_retitle(
12305 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12306 own,
12307 &leaked
12308 ));
12309 assert!(should_retitle(
12310 "chore: land candidate A of run 1",
12311 own,
12312 &leaked
12313 ));
12314 assert!(should_retitle(
12315 "magi: candidate A (uncommitted work)",
12316 own,
12317 &leaked
12318 ));
12319 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12320 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12321 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12322 }
12323
12324 #[test]
12325 fn pr_merge_plan_creates_adopts_or_stops() {
12326 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12327 assert_eq!(
12328 pr_merge_plan(Ok(land::OpenPr::One {
12329 url: "u".into(),
12330 title: "t".into()
12331 })),
12332 PrPlan::Adopt {
12333 url: "u".into(),
12334 title: "t".into()
12335 }
12336 );
12337 let PrPlan::Stop(many) =
12338 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12339 else {
12340 panic!("many must stop");
12341 };
12342 assert!(many.contains('a') && many.contains('b'));
12343 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12344 panic!("a failed lookup must stop");
12345 };
12346 assert!(err.contains("bad token"));
12347 }
12348
12349 use super::*;
12350 use crate::run::GateStatus;
12351 use std::collections::BTreeMap;
12352 use std::time::Duration;
12353
12354 fn conductor() -> AgentSpec {
12355 AgentSpec {
12356 id: "conductor".to_owned(),
12357 kind: crate::config::AgentKind::Command,
12358 model: None,
12359 command: vec!["true".to_owned()],
12360 extra_args: Vec::new(),
12361 env: BTreeMap::new(),
12362 prompt_delivery: None,
12363 max_turn_secs: None,
12364 }
12365 }
12366
12367 fn spec(id: &str) -> AgentSpec {
12368 AgentSpec {
12369 id: id.to_owned(),
12370 kind: crate::config::AgentKind::Command,
12371 model: None,
12372 command: vec!["true".to_owned()],
12373 extra_args: Vec::new(),
12374 env: BTreeMap::new(),
12375 prompt_delivery: None,
12376 max_turn_secs: None,
12377 }
12378 }
12379
12380 fn ids(xs: &[&str]) -> BTreeSet<String> {
12381 xs.iter().map(|s| (*s).to_owned()).collect()
12382 }
12383
12384 #[test]
12385 fn pick_successor_skips_an_agent_another_seat_holds() {
12386 let roster = [spec("a"), spec("b"), spec("c")];
12388 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12389 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12390 }
12391
12392 #[test]
12393 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12394 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12396 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12397 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12398 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12399 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12400 }
12401
12402 #[test]
12403 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12404 let roster = [spec("a"), spec("b")];
12405 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12406 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12407 let carried = ids(&["b"]);
12408 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12409 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12410 }
12411
12412 #[test]
12413 fn pick_successor_returns_none_without_any_untried_successor() {
12414 let roster = [spec("a"), spec("b")];
12415 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12416 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12417 let carried = ids(&["a"]);
12418 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12419 }
12420
12421 #[test]
12422 fn pick_successor_rescue_avoids_another_seats_occupant() {
12423 let roster = [spec("a"), spec("b"), spec("c")];
12425 let carried = ids(&["b", "c"]);
12426 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12427 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12428 }
12429
12430 #[test]
12431 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12432 let roster = [spec("a"), spec("b"), spec("c")];
12433 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12434 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12435 }
12436
12437 #[test]
12438 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12439 let roster = [spec("a"), spec("b"), spec("c")];
12440 let failed = ids(&["a", "b", "c"]);
12441 let mut tried = ids(&["b"]);
12443 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12444 assert_eq!(first.id, "a");
12445 tried.insert(first.id.clone());
12446 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12447 assert_eq!(second.id, "c");
12448 tried.insert(second.id.clone());
12449 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12450 }
12451
12452 #[test]
12453 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12454 let roster = [spec("a"), spec("b")];
12455 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12456 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12457 }
12458
12459 #[test]
12460 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12461 let roster = [spec("a"), spec("b"), spec("c")];
12462 let h = SeatHistory {
12463 failed: ids(&["a"]),
12464 last_ok: Some("b".to_owned()),
12465 last_fail: None,
12466 };
12467 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12468 let gone = SeatHistory {
12470 last_ok: Some("zzz".to_owned()),
12471 ..h.clone()
12472 };
12473 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12474 let failed = SeatHistory {
12475 failed: ids(&["a", "b"]),
12476 last_ok: Some("b".to_owned()),
12477 last_fail: None,
12478 };
12479 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12480 }
12481
12482 #[test]
12483 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12484 let first = SeatState::new("review-1", "alpha", 7);
12485 let next = handover_seat("review-1", "gamma", 7);
12486 assert_ne!(first.claude_session, next.claude_session);
12487 }
12488
12489 #[test]
12490 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12491 let mut state = state_with_summary("x", "y");
12492 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12493 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12494 assert_ne!(a.claude_session, b.claude_session);
12495 }
12496
12497 #[test]
12498 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12499 let roster = [spec("a"), spec("b")];
12500 let h = SeatHistory {
12501 failed: ids(&["a", "b"]),
12502 ..SeatHistory::default()
12503 };
12504 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12505 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12506 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12507 }
12508
12509 #[test]
12516 fn failure_signature_ignores_numbers_and_paths() {
12517 assert_eq!(
12518 failure_signature("exited with Some(2) and no usable output"),
12519 failure_signature("exited with Some(137) and no usable output")
12520 );
12521 assert_eq!(
12522 failure_signature("process exited with no usable output (exit code 2)"),
12523 failure_signature("process exited with no usable output (exit code 137)")
12524 );
12525 for (old, new) in [
12526 (
12527 "exited with Some(2) and no usable output",
12528 "process exited with no usable output (exit code 2)",
12529 ),
12530 (
12531 "exited with None and no usable output",
12532 "process exited with no usable output (exit code unavailable)",
12533 ),
12534 ] {
12535 let previous = FailClass::Other(failure_signature(old));
12536 let current = FailClass::Other(failure_signature(new));
12537 assert_eq!(previous, current, "resumption retains the failure class");
12538 assert!(!should_hand_over(Some(&previous), ¤t));
12539 }
12540 assert_eq!(
12541 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12542 failure_signature("cannot open /var/x.txt: denied")
12543 );
12544 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12545 }
12546
12547 #[test]
12548 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12549 let other = FailClass::Other("x".into());
12550 assert!(should_hand_over(None, &FailClass::Quota));
12551 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12552 assert!(should_hand_over(
12553 Some(&FailClass::Timeout),
12554 &FailClass::Timeout
12555 ));
12556 assert!(should_hand_over(None, &other));
12557 assert!(!should_hand_over(Some(&other), &other));
12558 assert!(should_hand_over(
12559 Some(&other),
12560 &FailClass::Other("y".into())
12561 ));
12562 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12564 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12565 }
12566
12567 #[test]
12568 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12569 let a = SeatState::new("judge-1", "alpha", 7);
12570 let b = handover_seat("judge-1", "beta", 7);
12571 assert_ne!(a.claude_session, b.claude_session);
12572 assert_eq!(b.turns, 0);
12573 }
12574
12575 #[test]
12576 fn a_timeout_is_classified_apart_from_other_failures() {
12577 assert_eq!(
12578 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12579 Some(FailClass::Timeout)
12580 );
12581 assert!(matches!(
12582 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12583 Some(FailClass::Other(_))
12584 ));
12585 }
12586
12587 #[test]
12588 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12589 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12590 let tried = BTreeSet::from(["beta".to_owned()]);
12591 let next = next_untried_in_roster(&roster, 1, &tried);
12594 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12595 }
12596
12597 #[test]
12598 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12599 let roster = vec![spec("alpha"), spec("beta")];
12600 let tried = BTreeSet::from(["beta".to_owned()]);
12601 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12605 }
12606
12607 #[test]
12608 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12609 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12610 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12611 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12615 }
12616
12617 #[test]
12618 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12619 let roster = vec![spec("a"), spec("a"), spec("b")];
12620 let tried = BTreeSet::from(["a".to_owned()]);
12621 let next = next_untried_in_roster(&roster, 0, &tried);
12622 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12623 }
12624
12625 #[test]
12626 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12627 let roster = vec![spec("a"), spec("b")];
12628 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12629 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12630 }
12631
12632 #[test]
12633 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12634 let dir = tempfile::tempdir().unwrap();
12635 let bay = dir.path().join("ffff");
12636
12637 remove_if_empty(&bay);
12639 assert!(!bay.exists());
12640
12641 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12644 remove_if_empty(&bay);
12645 assert!(bay.exists(), "non-empty directory must survive");
12646
12647 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12649 remove_if_empty(&bay);
12650 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12651 }
12652
12653 #[test]
12662 fn a_full_panel_that_found_nothing_is_clean() {
12663 assert!(round_is_clean(
12664 0,
12665 true,
12666 2,
12667 2,
12668 0,
12669 IncompleteReviewPolicy::Block
12670 ));
12671 }
12672
12673 #[test]
12674 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12675 assert!(!round_is_clean(
12676 0,
12677 true,
12678 1,
12679 2,
12680 0,
12681 IncompleteReviewPolicy::Block
12682 ));
12683 }
12684
12685 #[test]
12686 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12687 assert!(!round_is_clean(
12688 1,
12689 true,
12690 1,
12691 2,
12692 0,
12693 IncompleteReviewPolicy::Warn
12694 ));
12695 }
12696
12697 #[test]
12698 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12699 assert!(round_is_clean(
12700 0,
12701 true,
12702 1,
12703 2,
12704 0,
12705 IncompleteReviewPolicy::Warn
12706 ));
12707 }
12708
12709 #[test]
12710 fn a_full_panel_with_an_open_finding_is_not_clean() {
12711 assert!(!round_is_clean(
12712 1,
12713 true,
12714 2,
12715 2,
12716 0,
12717 IncompleteReviewPolicy::Block
12718 ));
12719 }
12720
12721 #[test]
12722 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12723 assert!(!round_is_clean(
12724 0,
12725 false,
12726 2,
12727 2,
12728 0,
12729 IncompleteReviewPolicy::Block
12730 ));
12731 }
12732
12733 #[test]
12740 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12741 assert!(round_is_clean(
12744 0,
12745 true,
12746 1,
12747 2,
12748 1,
12749 IncompleteReviewPolicy::Block
12750 ));
12751 }
12752
12753 #[test]
12754 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12755 assert!(!round_is_clean(
12758 0,
12759 true,
12760 1,
12761 2,
12762 0,
12763 IncompleteReviewPolicy::Block
12764 ));
12765 }
12766
12767 #[test]
12768 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12769 assert!(!round_is_clean(
12770 1,
12771 true,
12772 1,
12773 2,
12774 1,
12775 IncompleteReviewPolicy::Block
12776 ));
12777 assert!(!round_is_clean(
12778 0,
12779 false,
12780 1,
12781 2,
12782 1,
12783 IncompleteReviewPolicy::Block
12784 ));
12785 }
12786
12787 #[test]
12788 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12789 assert!(!round_is_clean(
12793 0,
12794 true,
12795 0,
12796 2,
12797 2,
12798 IncompleteReviewPolicy::Block
12799 ));
12800 }
12801
12802 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12803 CommandOutcome {
12804 command: "test".to_owned(),
12805 code,
12806 output_tail: String::new(),
12807 duration_ms: 0,
12808 resource_blocked,
12809 }
12810 }
12811
12812 #[test]
12813 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12814 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12815 assert!(
12816 !verify_inconclusive(&[outcome(Some(1), false)]),
12817 "an ordinary failure is still evidence about the patch"
12818 );
12819 assert!(verify_inconclusive(&[outcome(None, true)]));
12820 assert!(
12821 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12822 "one inconclusive outcome taints the whole batch"
12823 );
12824 assert!(!verify_inconclusive(&[]));
12825 }
12826
12827 #[tokio::test]
12828 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12829 let calls = std::sync::atomic::AtomicUsize::new(0);
12833 let started = Instant::now();
12834 wait_for_pids_with(
12835 &[123],
12836 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12837 Duration::from_millis(5),
12838 Duration::from_secs(5),
12839 )
12840 .await;
12841 assert!(
12842 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12843 "must keep checking rather than deciding on the first answer"
12844 );
12845 assert!(
12846 started.elapsed() < Duration::from_secs(1),
12847 "must return the moment it is confirmed dead, not wait out the ceiling"
12848 );
12849 }
12850
12851 #[tokio::test]
12852 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12853 let started = Instant::now();
12854 wait_for_pids_with(
12855 &[123],
12856 |_| true, Duration::from_millis(5),
12858 Duration::from_millis(30),
12859 )
12860 .await;
12861 let elapsed = started.elapsed();
12862 assert!(
12863 elapsed >= Duration::from_millis(30),
12864 "must not give up before its own ceiling: {elapsed:?}"
12865 );
12866 assert!(
12867 elapsed < Duration::from_secs(1),
12868 "must not wait past its own ceiling either: {elapsed:?}"
12869 );
12870 }
12871
12872 #[tokio::test]
12873 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12874 let started = Instant::now();
12875 wait_for_pids_with(
12876 &[],
12877 |_| true,
12878 Duration::from_secs(5),
12879 Duration::from_secs(5),
12880 )
12881 .await;
12882 assert!(
12883 started.elapsed() < Duration::from_millis(200),
12884 "an empty pid list has nothing to confirm"
12885 );
12886 }
12887
12888 fn review_round(
12894 clean: bool,
12895 blocking: usize,
12896 answered: usize,
12897 expected: usize,
12898 progressed: bool,
12899 e2e_ok: bool,
12900 ) -> ReviewRound {
12901 ReviewRound {
12902 round: 1,
12903 head: "h".to_owned(),
12904 verified_head: None,
12905 verified_at: None,
12906 reviews: Vec::new(),
12907 e2e: vec![CommandOutcome {
12908 command: "test".to_owned(),
12909 code: Some(if e2e_ok { 0 } else { 1 }),
12910 output_tail: String::new(),
12911 duration_ms: 0,
12912 resource_blocked: false,
12913 }],
12914 verify_retried: false,
12915 e2e_deferred: false,
12916 e2e_defer_reason: None,
12917 fix: None,
12918 blocking,
12919 answered,
12920 expected,
12921 clean,
12922 progressed,
12923 vote_split: false,
12924 reconsideration: Vec::new(),
12925 verdict: None,
12926 }
12927 }
12928
12929 #[test]
12930 fn review_conclusion_is_none_when_nothing_has_run() {
12931 assert_eq!(review_conclusion(&[], 3), None);
12932 }
12933
12934 #[test]
12935 fn review_conclusion_is_none_while_rounds_remain() {
12936 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12937 assert_eq!(review_conclusion(&rounds, 3), None);
12938 }
12939
12940 #[test]
12941 fn review_conclusion_is_gating_once_a_round_is_clean() {
12942 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12943 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12944 }
12945
12946 #[test]
12947 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12948 let rounds = vec![
12949 review_round(false, 1, 2, 2, true, true),
12950 review_round(false, 1, 2, 2, true, true),
12951 ];
12952 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12953 }
12954
12955 #[test]
12956 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12957 let rounds = vec![
12958 review_round(false, 1, 2, 2, true, true),
12959 review_round(false, 1, 2, 2, true, false),
12960 ];
12961 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12962 }
12963
12964 #[test]
12965 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12966 let mut blocked = review_round(false, 1, 2, 2, true, false);
12973 blocked.e2e[0].resource_blocked = true;
12974 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12975 assert_eq!(review_conclusion(&rounds, 2), None);
12976 }
12977
12978 #[test]
12979 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12980 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12982 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12983 }
12984
12985 #[test]
12986 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12987 let rounds = vec![
12988 review_round(false, 1, 2, 2, false, true),
12989 review_round(false, 1, 2, 2, false, true),
12990 ];
12991 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12992 }
12993
12994 fn secs(n: u64) -> Duration {
12995 Duration::from_secs(n)
12996 }
12997
12998 #[tokio::test]
13001 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13002 ask_test_home();
13003 let tmp = tempfile::tempdir().unwrap();
13004 let repo = tmp.path().join("repo");
13005 std::fs::create_dir_all(&repo).unwrap();
13006 init_repo(&repo);
13007 let g = |args: &[&str]| {
13008 let out = std::process::Command::new("git")
13009 .args(args)
13010 .current_dir(&repo)
13011 .quiet()
13012 .output()
13013 .unwrap();
13014 assert!(
13015 out.status.success(),
13016 "{}",
13017 String::from_utf8_lossy(&out.stderr)
13018 );
13019 };
13020 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13021 g(&["checkout", "-b", "historical"]);
13022 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13023 g(&["add", "."]);
13024 g(&["commit", "-m", "old alternative"]);
13025 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13026 g(&["checkout", "-b", "magi/task-selected", &base]);
13027 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13028 g(&["add", "."]);
13029 g(&["commit", "-m", "measurement infrastructure"]);
13030 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13031 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13032 g(&["add", "."]);
13033 g(&["commit", "-m", "dependent implementation"]);
13034 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13035 g(&["checkout", "main"]);
13036 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13037 g(&["add", "."]);
13038 g(&["commit", "-m", "unrelated base change"]);
13039 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13040 let origin = tmp.path().join("origin.git");
13041 g(&[
13042 "clone",
13043 "--bare",
13044 repo.to_str().unwrap(),
13045 origin.to_str().unwrap(),
13046 ]);
13047 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13048 g(&["fetch", "origin"]);
13049 let selection = crate::pr_recovery::Selection {
13050 url: "https://example.test/project/pull/1".into(),
13051 branch: "magi/task-selected".into(),
13052 head: head.clone(),
13053 source_head: None,
13054 source_run: None,
13055 source_proof: None,
13056 disposition: Some(crate::pr_recovery::Disposition::Review),
13057 instruction: "preserve measured behavior".into(),
13058 approved_at: jiff::Timestamp::now(),
13059 };
13060 let mut template = runner_at(RunStatus::Reviewing);
13061 template.state.repo = repo.clone();
13062 template.state.base_commit = current.clone();
13063 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13064 template.state.origin = Some(crate::run::Origin::queue("selected"));
13065 template.state.pr_head = Some(selection.branch.clone());
13066 template.state.pr_recovery = Some(selection.clone());
13067 let mut runner =
13068 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13069 .await
13070 .unwrap();
13071 let wt = runner.state.winner().unwrap().worktree.clone();
13072 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13073 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13074 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13075 assert!(runner.state.seeds.is_empty());
13076 assert_eq!(runner.state.instruction, "preserve measured behavior");
13077 runner.sync_to_base().await.unwrap();
13078 assert_eq!(
13079 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13080 "prerequisite"
13081 );
13082 assert_eq!(
13083 std::fs::read_to_string(wt.join("implementation")).unwrap(),
13084 "measured implementation"
13085 );
13086 assert_eq!(
13087 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13088 "current main change"
13089 );
13090 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13091 assert_eq!(
13092 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13093 head
13094 );
13095
13096 let mut partial = runner_at(RunStatus::Reviewing);
13098 partial.state.repo = repo.clone();
13099 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13100 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13101 run: Some("exit 1".into()),
13102 ..Default::default()
13103 }];
13104 let selection = runner.state.pr_recovery.clone().unwrap();
13105 partial.state.pr_recovery = Some(selection.clone());
13106 partial.state.pr_head = Some(selection.branch.clone());
13107 partial.state.origin = Some(crate::run::Origin::queue("selected"));
13108 let id = partial.state.id.clone();
13109 let error = match Runner::open_selected(
13110 &repo,
13111 partial.state,
13112 partial.roles,
13113 1,
13114 runner.state.base_commit.clone(),
13115 selection,
13116 )
13117 .await
13118 {
13119 Ok(_) => panic!("setup must fail"),
13120 Err(e) => e,
13121 };
13122 assert_eq!(
13123 error
13124 .downcast_ref::<crate::pr_recovery::StartupFailed>()
13125 .unwrap()
13126 .run,
13127 id
13128 );
13129 let failed = RunState::load(&id).unwrap();
13130 assert_eq!(failed.status, RunStatus::Blocked);
13131 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13132 assert!(Runner::resume(&id).is_err());
13133 runner.state.status = RunStatus::Ready;
13134 runner.state.driver_exited = true;
13135 runner.state.save().unwrap();
13136 let mut retry = runner_at(RunStatus::Reviewing);
13137 retry.state.repo = repo.clone();
13138 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13139 let mut reselected = runner.state.pr_recovery.clone().unwrap();
13140 reselected.approved_at = jiff::Timestamp::now();
13141 retry.state.origin = Some(crate::run::Origin::queue("selected"));
13142 retry.state.pr_head = Some(reselected.branch.clone());
13143 retry.state.pr_recovery = Some(reselected.clone());
13144 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13145 .await
13146 .unwrap();
13147 let restarted = Runner::open_selected(
13148 &repo,
13149 retry.state,
13150 retry.roles,
13151 1,
13152 runner.state.base_commit.clone(),
13153 reselected,
13154 )
13155 .await
13156 .unwrap();
13157 assert_eq!(
13158 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13159 .await
13160 .unwrap(),
13161 head
13162 );
13163 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13164 assert_eq!(
13165 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13166 head
13167 );
13168 }
13169
13170 #[tokio::test]
13171 #[cfg(unix)]
13172 async fn fallback_and_interrupted_restoration_require_coordination() {
13173 use std::os::fd::AsRawFd as _;
13174 ask_test_home();
13175 let tmp = tempfile::tempdir().unwrap();
13176 let mut runner = registered_runner(tmp.path()).await;
13177 runner.state.config.graph.timeout_resource_wait = 1;
13178 let winner = runner.state.candidates[0].clone();
13179 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13180 let leftover = winner.worktree.join("leftover");
13181 std::fs::write(&leftover, "untrusted edits").unwrap();
13182 let next = tmp.path().join("next-fixer");
13183 let mut failed = spec("first");
13184 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13185 let mut successor = spec("second");
13186 successor.command = vec![
13187 "sh".into(),
13188 "-c".into(),
13189 format!("touch '{}'; echo done", next.display()),
13190 ];
13191 let attempts = vec![
13192 (failed.clone(), "first".into()),
13193 (successor.clone(), "second".into()),
13194 ];
13195 let lock_path = runner.state.repo.join(".git/updater.lock");
13196 let external = std::fs::File::create(&lock_path).unwrap();
13197 assert_eq!(
13198 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13199 0
13200 );
13201 let artifacts = tmp.path().join("artifacts");
13202 let build = |spec, seat| SeatJob {
13203 spec,
13204 seat,
13205 cwd: winner.worktree.clone(),
13206 prompt: "fixture".into(),
13207 timeout: Duration::from_secs(10),
13208 allow_write: true,
13209 sessions: false,
13210 artifacts: artifacts.clone(),
13211 stem: "fix".into(),
13212 handover: None,
13213 };
13214 let (_, _, out) = runner
13215 .ask_fixer_among(
13216 attempts.clone(),
13217 "graft",
13218 None,
13219 Some((&winner.worktree, &head)),
13220 build,
13221 )
13222 .await;
13223 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13224 assert!(leftover.exists(), "contention must prevent clean");
13225 assert!(!next.exists());
13226 assert!(runner.state.handovers.is_empty());
13227 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13229 from_run: "earlier".into(),
13230 generation: 1,
13231 mode: crate::recovery::Mode::VerifyRepair,
13232 reasons: Vec::new(),
13233 reviewed_head: None,
13234 rounds: Vec::new(),
13235 open_findings: Vec::new(),
13236 evidence: None,
13237 criteria_inherited: true,
13238 repair: crate::recovery::Repair::Started {
13239 before: head.clone(),
13240 },
13241 });
13242 assert!(
13243 runner
13244 .recovery_repair(&winner)
13245 .await
13246 .unwrap_err()
13247 .is::<crate::repo_lock::ResourceBlocked>()
13248 );
13249 assert!(matches!(
13250 runner.state.recovery.as_ref().unwrap().repair,
13251 crate::recovery::Repair::Started { .. }
13252 ));
13253 assert!(leftover.exists());
13254 drop(external);
13255 runner.recovery_repair(&winner).await.unwrap();
13256 assert!(!leftover.exists());
13257 assert!(matches!(
13258 runner.state.recovery.as_ref().unwrap().repair,
13259 crate::recovery::Repair::Done { .. }
13260 ));
13261 std::fs::remove_file(&lock_path).unwrap();
13263 std::fs::create_dir(&lock_path).unwrap();
13264 std::fs::write(&leftover, "untrusted edits").unwrap();
13265 let (_, _, out) = runner
13266 .ask_fixer_among(
13267 attempts,
13268 "graft",
13269 None,
13270 Some((&winner.worktree, &head)),
13271 build,
13272 )
13273 .await;
13274 assert!(matches!(out, AgentOutcome::Failed(_)));
13275 assert!(leftover.exists());
13276 assert!(!next.exists());
13277 assert_eq!(
13278 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13279 head
13280 );
13281 }
13282
13283 #[cfg(unix)]
13284 async fn registered_runner(root: &Path) -> Runner {
13285 ask_test_home();
13286 let repo = root.join("repo");
13287 std::fs::create_dir_all(&repo).unwrap();
13288 init_repo(&repo);
13289 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13290 let mut runner = runner_at(RunStatus::Prep);
13291 runner.state.repo = repo.clone();
13292 runner.state.config.coordination = crate::config::test_coordination();
13293 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13294 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13295 runner.state.config.graph.advise = false;
13296 runner.roles.implementers = vec![spec("implement")];
13297 runner.roles.judges = vec![spec("judge")];
13298 runner.prep().await.unwrap();
13299 runner
13300 }
13301
13302 #[tokio::test]
13303 #[cfg(unix)]
13304 async fn parked_implementer_persists_completed_work() {
13305 for registered in [false, true] {
13306 let tmp = tempfile::tempdir().unwrap();
13307 let mut runner = registered_runner(tmp.path()).await;
13308 if !registered {
13309 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13310 }
13311 let marker = tmp.path().join("started");
13312 let release = tmp.path().join("release");
13313 let mut implementer = spec("implement");
13314 implementer.command = vec![
13315 "sh".into(),
13316 "-c".into(),
13317 format!(
13318 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13319 marker.display(),
13320 release.display()
13321 ),
13322 ];
13323 runner.roles.implementers = vec![implementer.clone()];
13324 runner.state.config.agents = vec![implementer];
13325 let pause = runner.pause.clone();
13326 let (result, ()) = tokio::join!(runner.implement(), async {
13327 tokio::time::timeout(Duration::from_secs(10), async {
13328 while !marker.exists() {
13329 tokio::time::sleep(Duration::from_millis(10)).await;
13330 }
13331 })
13332 .await
13333 .unwrap();
13334 pause.park();
13335 std::fs::write(&release, "go").unwrap();
13336 });
13337 result.unwrap();
13338 let candidate = &runner.state.candidates[0];
13339 assert!(candidate.commits > 0, "{candidate:?}");
13340 assert!(candidate.summary.contains("saved-result"));
13341 assert_eq!(candidate.files, 1);
13342 let before = candidate.commits;
13343 runner.implement().await.unwrap();
13344 assert_eq!(runner.state.candidates[0].commits, before);
13345 }
13346 }
13347
13348 #[tokio::test]
13349 #[cfg(unix)]
13350 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13351 let tmp = tempfile::tempdir().unwrap();
13352 let mut runner = registered_runner(tmp.path()).await;
13353 let winner = runner.state.candidates[0].clone();
13354 let cache = tmp.path().join("cache");
13355 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13356 runner.state.config.graph.timeout_resource_wait = 0;
13357 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13358 let crate::cache::AcquireOutcome::Acquired(held) =
13359 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13360 else {
13361 panic!("cache must be free")
13362 };
13363 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13364 from_run: "earlier".into(),
13365 generation: 1,
13366 mode: crate::recovery::Mode::VerifyRepair,
13367 reasons: Vec::new(),
13368 reviewed_head: None,
13369 rounds: Vec::new(),
13370 open_findings: Vec::new(),
13371 evidence: None,
13372 criteria_inherited: true,
13373 repair: crate::recovery::Repair::NotStarted,
13374 });
13375 runner.recovery_repair(&winner).await.unwrap();
13376 assert!(matches!(
13377 runner.state.recovery.as_ref().unwrap().repair,
13378 crate::recovery::Repair::WaitingForRestore { .. }
13379 ));
13380 drop(held);
13381 runner.state.resource_block = None;
13382 runner.state.config.graph.timeout_resource_wait = 1;
13383 runner.recovery_repair(&winner).await.unwrap();
13384 assert!(matches!(
13385 runner.state.recovery.as_ref().unwrap().repair,
13386 crate::recovery::Repair::Done { .. }
13387 ));
13388 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13389 }
13390
13391 #[tokio::test]
13392 #[cfg(unix)]
13393 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13394 use std::os::fd::AsRawFd as _;
13395 for graft in [false, true] {
13396 let tmp = tempfile::tempdir().unwrap();
13397 let mut runner = registered_runner(tmp.path()).await;
13398 runner.state.config.graph.timeout_resource_wait = 1;
13399 let winner = runner.state.candidates[0].clone();
13400 let marker = tmp.path().join("fixer-started");
13401 let mut fixer = spec("fixer");
13402 fixer.command = vec![
13403 "sh".into(),
13404 "-c".into(),
13405 format!("echo started >> '{}'; echo done", marker.display()),
13406 ];
13407 runner.roles.fixer = Some(vec![fixer]);
13408 if graft {
13409 let mut loser = winner.clone();
13410 loser.label = 'B';
13411 loser.index = 1;
13412 runner.state.candidates.push(loser);
13413 runner.state.config.graph.graft_runner_up = true;
13414 runner.state.tally = Some(Tally {
13415 first_choice: BTreeMap::new(),
13416 borda: BTreeMap::new(),
13417 winner: winner.label,
13418 rankings: 1,
13419 unanimous_initial: true,
13420 deliberated: false,
13421 changed_votes: 0,
13422 unanimous_final: true,
13423 tie_break: None,
13424 judges: 1,
13425 present: 1,
13426 quorum: 1,
13427 met_quorum: true,
13428 uncontested: None,
13429 });
13430 runner.state.judgements.push(crate::run::Judgement {
13431 judge: 1,
13432 seat: "judge-1".into(),
13433 agent: "judge".into(),
13434 ranking: vec![winner.label, 'B'],
13435 reasons: BTreeMap::new(),
13436 confidence: None,
13437 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13438 order: vec![0, 1],
13439 failed: None,
13440 duration_ms: 0,
13441 });
13442 } else {
13443 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13444 from_run: "earlier".into(),
13445 generation: 1,
13446 mode: crate::recovery::Mode::VerifyRepair,
13447 reasons: Vec::new(),
13448 reviewed_head: None,
13449 rounds: Vec::new(),
13450 open_findings: Vec::new(),
13451 evidence: None,
13452 criteria_inherited: true,
13453 repair: crate::recovery::Repair::NotStarted,
13454 });
13455 }
13456 let reader = crate::repo_lock::SeatLease::for_checkout(
13457 &winner.worktree,
13458 false,
13459 crate::config::test_coordination().as_ref(),
13460 )
13461 .await
13462 .unwrap();
13463 let pause = runner.pause.clone();
13464 let (result, ()) = tokio::join!(
13465 async {
13466 if graft {
13467 runner.graft().await
13468 } else {
13469 runner.recovery_repair(&winner).await
13470 }
13471 },
13472 async {
13473 tokio::time::sleep(Duration::from_millis(100)).await;
13474 pause.park();
13475 }
13476 );
13477 result.unwrap();
13478 assert!(!marker.exists(), "a blocked seat must not spawn");
13479 runner.state = RunState::load(&runner.state.id).unwrap();
13481 if graft {
13482 assert_eq!(
13483 runner.state.graft.as_ref().unwrap().outcome,
13484 GraftOutcome::WaitingForRestore
13485 );
13486 } else {
13487 assert!(matches!(
13488 runner.state.recovery.as_ref().unwrap().repair,
13489 crate::recovery::Repair::WaitingForRestore { .. }
13490 ));
13491 }
13492 drop(reader);
13493 runner.pause = Pause::new();
13494 runner.state.resource_block = None;
13495 let external =
13497 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13498 assert_eq!(
13499 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13500 0
13501 );
13502 let leftover = winner.worktree.join("leftover");
13503 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13504 let result = if graft {
13505 runner.graft().await
13506 } else {
13507 runner.recovery_repair(&winner).await
13508 };
13509 assert!(
13510 result
13511 .unwrap_err()
13512 .is::<crate::repo_lock::ResourceBlocked>()
13513 );
13514 assert!(leftover.exists());
13515 assert!(!marker.exists());
13516 drop(external);
13517 runner.state.resource_block = None;
13518 let result = if graft {
13519 runner.graft().await
13520 } else {
13521 runner.recovery_repair(&winner).await
13522 };
13523 result.unwrap();
13524 assert!(!leftover.exists(), "restore must precede retry");
13525 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13526 if graft {
13527 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13528 } else {
13529 assert!(matches!(
13530 runner.state.recovery.as_ref().unwrap().repair,
13531 crate::recovery::Repair::Done { .. }
13532 ));
13533 }
13534 }
13535 }
13536
13537 #[tokio::test]
13538 #[cfg(unix)]
13539 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13540 use std::os::fd::AsRawFd as _;
13541 let tmp = tempfile::tempdir().unwrap();
13542 let mut runner = registered_runner(tmp.path()).await;
13543 runner.state.config.graph.timeout_resource_wait = 1;
13544 let winner = runner.state.candidates[0].clone();
13545 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13546 runner.state.tally = Some(Tally {
13547 first_choice: BTreeMap::new(),
13548 borda: BTreeMap::new(),
13549 winner: winner.label,
13550 rankings: 0,
13551 unanimous_initial: false,
13552 deliberated: false,
13553 changed_votes: 0,
13554 unanimous_final: false,
13555 tie_break: None,
13556 judges: 0,
13557 present: 0,
13558 quorum: 0,
13559 met_quorum: true,
13560 uncontested: Some("fixture".into()),
13561 });
13562 runner.state.graft = Some(GraftRecord {
13563 pre: pre.clone(),
13564 post: None,
13565 lines: None,
13566 limit: None,
13567 agent: None,
13568 outcome: GraftOutcome::Running,
13569 requested: Vec::new(),
13570 adoption: None,
13571 adoption_error: None,
13572 });
13573 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13574 git::commit_all(&winner.worktree, "unfinished graft")
13575 .await
13576 .unwrap();
13577 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13578 let leftover = winner.worktree.join("leftover");
13579 std::fs::write(&leftover, "untrusted edits").unwrap();
13580 let path = runner.state.repo.join(".git/updater.lock");
13581 let external = std::fs::File::create(&path).unwrap();
13582 assert_eq!(
13583 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13584 0
13585 );
13586 assert!(
13587 runner
13588 .graft()
13589 .await
13590 .unwrap_err()
13591 .is::<crate::repo_lock::ResourceBlocked>()
13592 );
13593 assert!(matches!(
13594 runner.state.graft.as_ref().unwrap().outcome,
13595 GraftOutcome::Running
13596 ));
13597 assert_eq!(
13598 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13599 unfinished
13600 );
13601 assert!(leftover.exists());
13602 drop(external);
13603 std::fs::remove_file(&path).unwrap();
13604 std::fs::create_dir(&path).unwrap();
13605 assert!(runner.graft().await.is_err());
13606 assert!(matches!(
13607 runner.state.graft.as_ref().unwrap().outcome,
13608 GraftOutcome::Running
13609 ));
13610 assert_eq!(
13611 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13612 unfinished
13613 );
13614 assert!(leftover.exists());
13615 std::fs::remove_dir(&path).unwrap();
13616 runner.graft().await.unwrap();
13617 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13618 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13619 assert!(!leftover.exists());
13620 }
13621
13622 #[tokio::test]
13623 #[cfg(unix)]
13624 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13625 use std::os::fd::AsRawFd as _;
13626 let tmp = tempfile::tempdir().unwrap();
13627 let mut runner = registered_runner(tmp.path()).await;
13628 runner.state.prep_complete = Some(false);
13629 runner.state.config.graph.timeout_resource_wait = 1;
13630 runner.state.save().unwrap();
13631 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13632 assert_eq!(
13633 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13634 0
13635 );
13636 runner.execute().await.unwrap();
13637 let disk = RunState::load(&runner.state.id).unwrap();
13638 assert!(disk.driver_exited);
13639 assert_eq!(disk.driver_pid, None);
13640 assert_eq!(runner.state.status, RunStatus::Blocked);
13641 assert_eq!(
13642 runner.state.resource_block.as_ref().unwrap().kind,
13643 "repository coordination"
13644 );
13645 assert!(runner.state.prep_failure.is_none());
13646 assert!(runner.state.seats.is_empty());
13647 runner.pause.park();
13648 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13651 .await
13652 .unwrap()
13653 .unwrap();
13654 assert!(runner.state.parked);
13655 assert!(runner.state.prep_failure.is_none());
13656 let disk = RunState::load(&runner.state.id).unwrap();
13657 assert!(disk.driver_exited);
13658 assert_eq!(disk.driver_pid, None);
13659 }
13660
13661 #[tokio::test]
13662 #[cfg(unix)]
13663 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13664 use std::os::fd::AsRawFd as _;
13665 let tmp = tempfile::tempdir().unwrap();
13666 let mut runner = registered_runner(tmp.path()).await;
13667 runner.state.prep_complete = Some(false);
13668 runner.state.config.graph.timeout_resource_wait = 30;
13669 runner.state.save().unwrap();
13670 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13671 assert_eq!(
13672 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13673 0
13674 );
13675 assert!(
13676 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13677 .await
13678 .is_err()
13679 );
13680 let disk = RunState::load(&runner.state.id).unwrap();
13681 assert!(disk.driver_exited);
13682 assert_eq!(disk.driver_pid, None);
13683 assert!(disk.status.resumable());
13684 assert!(runner.state.driver_exited);
13685 assert_eq!(runner.state.driver_pid, None);
13686 runner.state.save().unwrap();
13687 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13688 }
13689
13690 #[tokio::test]
13691 #[cfg(unix)]
13692 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13693 let tmp = tempfile::tempdir().unwrap();
13694 let mut runner = registered_runner(tmp.path()).await;
13695 let winner = runner.state.candidates[0].clone();
13696 let cache = tmp.path().join("cache");
13697 let started = tmp.path().join("pre-gate-started");
13698 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13699 runner.state.config.verify.gate =
13700 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13701 let home = crate::run::home();
13702 let owner =
13703 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13704 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13705 crate::cache::AcquireOutcome::Acquired(g) => g,
13706 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13707 };
13708 let mut gate = Box::pin(runner.run_gate(&winner));
13709 assert!(
13710 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13711 .await
13712 .is_err()
13713 );
13714 assert!(!started.exists(), "pre-gate must wait for the cache too");
13715 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13719 let editor = crate::repo_lock::SeatLease::checkout_with(
13720 &winner.worktree,
13721 true,
13722 crate::config::test_coordination().as_ref(),
13723 &wait,
13724 )
13725 .await
13726 .unwrap();
13727 drop(editor);
13728 guard.release();
13729 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13730 .await
13731 .unwrap()
13732 .unwrap();
13733 drop(gate);
13734 assert!(outcomes.iter().all(CommandOutcome::ok));
13735 assert!(started.exists());
13736 }
13737
13738 #[tokio::test]
13739 #[cfg(unix)]
13740 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13741 use std::os::fd::AsRawFd as _;
13742
13743 ask_test_home();
13744 let tmp = tempfile::tempdir().unwrap();
13745 let repo = tmp.path().join("repo");
13746 std::fs::create_dir_all(&repo).unwrap();
13747 init_repo(&repo);
13748 let common = repo.join(".git");
13749 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13750 let mut runner = runner_at(RunStatus::Prep);
13751 runner.state.repo = repo.clone();
13752 runner.state.config.coordination = crate::config::test_coordination();
13753 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13754 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13755 runner.state.config.graph.advise = false;
13756 runner.roles.implementers = vec![spec("implement")];
13757 runner.roles.judges = vec![spec("judge")];
13758 runner.prep().await.unwrap();
13759 let winner = runner.state.candidates[0].clone();
13760 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13763 let completed = tmp.path().join("seat-completed");
13764 runner.roles.implementers[0].command = vec![
13765 "sh".into(),
13766 "-c".into(),
13767 format!("touch '{}'; echo completed", completed.display()),
13768 ];
13769 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13770 assert_eq!(
13771 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13772 0
13773 );
13774 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13775 let mut implement = Box::pin(runner.implement());
13776 tokio::select! {
13777 result = &mut implement => panic!("rescue must wait: {result:?}"),
13778 result = tokio::time::timeout(Duration::from_secs(10), async {
13779 while !completed.exists() {
13780 tokio::time::sleep(Duration::from_millis(20)).await;
13781 }
13782 }) => result.unwrap(),
13783 }
13784 assert!(
13785 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13786 .await
13787 .is_err()
13788 );
13789 assert_eq!(
13790 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13791 before
13792 );
13793 assert_eq!(
13794 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13795 0
13796 );
13797 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13798 .await
13799 .unwrap()
13800 .unwrap();
13801 drop(implement);
13802 assert_ne!(
13803 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13804 before
13805 );
13806 assert!(git::is_clean(&winner.worktree).await.unwrap());
13807
13808 let started = tmp.path().join("pre-gate-started");
13809 let proceed = tmp.path().join("pre-gate-proceed");
13810 let checked = tmp.path().join("gate-ran");
13811 runner.state.config.verify.pre_gate = vec![format!(
13812 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13813 started.display(),
13814 proceed.display()
13815 )];
13816 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13817 assert_eq!(
13818 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13819 0
13820 );
13821 let mut gate = Box::pin(runner.run_gate(&winner));
13822 assert!(
13823 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13824 .await
13825 .is_err()
13826 );
13827 assert!(!started.exists());
13828 assert!(!checked.exists());
13829 assert!(git::is_clean(&winner.worktree).await.unwrap());
13830 assert_eq!(
13831 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13832 0
13833 );
13834 tokio::select! {
13835 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13836 result = tokio::time::timeout(Duration::from_secs(10), async {
13837 while !started.exists() {
13838 tokio::time::sleep(Duration::from_millis(20)).await;
13839 }
13840 }) => result.unwrap(),
13841 }
13842 assert_eq!(
13843 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13844 -1,
13845 "mechanical edits must hold the real lock"
13846 );
13847 std::fs::write(proceed, "ready").unwrap();
13848 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13849 .await
13850 .unwrap()
13851 .unwrap();
13852 drop(gate);
13853 assert!(outcomes.iter().all(CommandOutcome::ok));
13854 assert!(checked.exists());
13855 assert!(runner.state.pre_gate_commit.is_some());
13856 assert!(git::is_clean(&winner.worktree).await.unwrap());
13857 assert_eq!(
13858 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13859 0,
13860 "gate completion must release coordination"
13861 );
13862 drop(outsider);
13863 std::fs::remove_file(common.join("updater.lock")).unwrap();
13864 std::fs::create_dir(common.join("updater.lock")).unwrap();
13865 std::fs::remove_file(&started).unwrap();
13866 std::fs::remove_file(&checked).unwrap();
13867 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13868 let error = runner.run_gate(&winner).await.unwrap_err();
13869 assert!(
13870 error
13871 .to_string()
13872 .contains("acquire mandatory repository lock")
13873 );
13874 assert!(
13875 !started.exists(),
13876 "failed acquisition must prevent pre-gate"
13877 );
13878 assert!(!checked.exists(), "failed acquisition must prevent gate");
13879 assert!(git::is_clean(&winner.worktree).await.unwrap());
13880 assert_eq!(
13881 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13882 head
13883 );
13884 }
13885
13886 #[tokio::test]
13887 #[cfg(unix)]
13888 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13889 use std::os::fd::AsRawFd as _;
13890 use std::time::Duration;
13891
13892 ask_test_home();
13893 let tmp = tempfile::tempdir().unwrap();
13894 let repo = tmp.path().join("repo");
13895 std::fs::create_dir_all(&repo).unwrap();
13896 init_repo(&repo);
13897 let common = repo.join(".git");
13898 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13899 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13900 assert_eq!(
13901 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13902 0
13903 );
13904 let mut runner = runner_at(RunStatus::Prep);
13905 runner.state.repo = repo.clone();
13906 runner.state.config.coordination = crate::config::test_coordination();
13907 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13908 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13909 runner.state.config.graph.advise = true;
13910 runner.state.config.graph.advisors = 1;
13911 runner.roles.implementers = vec![spec("implement")];
13912 runner.roles.judges = vec![spec("judge")];
13913 let root = runner.state.worktree_root();
13914 let started = tmp.path().join("setup-started");
13915 let proceed = tmp.path().join("setup-proceed");
13916 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13917 run: Some(format!(
13918 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13919 started.display(),
13920 proceed.display()
13921 )),
13922 ..Default::default()
13923 }];
13924 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13925 let config = git::git(&repo, &["config", "--local", "--list"])
13926 .await
13927 .unwrap();
13928 let mut prep = Box::pin(runner.prep());
13929 assert!(
13930 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13931 .await
13932 .is_err()
13933 );
13934 assert!(
13935 !root.exists(),
13936 "contention must prevent even checkout creation"
13937 );
13938 assert!(!started.exists(), "contention must prevent setup");
13939 assert_eq!(
13940 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13941 branches
13942 );
13943 assert_eq!(
13944 git::git(&repo, &["config", "--local", "--list"])
13945 .await
13946 .unwrap(),
13947 config
13948 );
13949 drop(lock);
13950 tokio::select! {
13951 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13952 result = tokio::time::timeout(Duration::from_secs(10), async {
13953 while !started.exists() {
13954 tokio::time::sleep(Duration::from_millis(20)).await;
13955 }
13956 }) => result.unwrap(),
13957 }
13958 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13959 assert_eq!(
13960 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13961 -1
13962 );
13963 assert!(
13964 tokio::time::timeout(
13965 Duration::from_millis(100),
13966 crate::repo_lock::SeatLease::for_checkout(
13967 &repo,
13968 true,
13969 crate::config::test_coordination().as_ref()
13970 )
13971 )
13972 .await
13973 .is_err(),
13974 "preparation must also exclude local editors"
13975 );
13976 std::fs::write(proceed, "ready").unwrap();
13977 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13978 .await
13979 .unwrap()
13980 .unwrap();
13981 drop(prep);
13982 assert_eq!(runner.state.prep_complete, Some(true));
13983 assert!(root.join("judge-1").exists());
13984 assert!(root.join("advisor-1").exists());
13985 assert_eq!(
13986 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13987 0
13988 );
13989 }
13990
13991 #[tokio::test]
13992 #[cfg(unix)]
13993 async fn preparation_lock_failure_prevents_repository_changes() {
13994 ask_test_home();
13995 let tmp = tempfile::tempdir().unwrap();
13996 let repo = tmp.path().join("repo");
13997 std::fs::create_dir_all(&repo).unwrap();
13998 init_repo(&repo);
13999 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14000 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14003 let mut runner = runner_at(RunStatus::Prep);
14004 runner.state.repo = repo.clone();
14005 runner.state.config.coordination = crate::config::test_coordination();
14006 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14007 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14008 runner.roles.implementers = vec![spec("implement")];
14009 let error = runner.prep().await.unwrap_err();
14010 assert!(
14011 error
14012 .to_string()
14013 .contains("acquire mandatory repository lock")
14014 );
14015 assert!(!runner.state.worktree_root().exists());
14016 assert!(runner.state.candidates.is_empty());
14017 assert!(!repo.join(".git/config.worktree").exists());
14018 assert_eq!(
14019 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14020 "* main"
14021 );
14022 }
14023
14024 #[tokio::test]
14025 async fn preparation_records_creation_intent_before_git_can_fail() {
14026 ask_test_home();
14027 let tmp = tempfile::tempdir().unwrap();
14028 let repo = tmp.path().join("repo");
14029 std::fs::create_dir_all(&repo).unwrap();
14030 init_repo(&repo);
14031 let mut runner = runner_at(RunStatus::Prep);
14032 runner.state.repo = repo.clone();
14033 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14034 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14035 runner.state.config.blind.commit_msg_hook = false;
14036 runner.roles.implementers = vec![spec("one")];
14037 let label = blind::assign_labels(1, runner.state.seed)[0];
14038 let branch = runner.state.branch_for(label);
14039 git::git(&repo, &["branch", &branch]).await.unwrap();
14042 assert!(runner.prep().await.is_err());
14043 let saved = RunState::load(&runner.state.id).unwrap();
14044 assert!(saved.candidates.is_empty());
14045 let event = saved
14046 .events
14047 .iter()
14048 .find(|e| e.node == "prep-intent")
14049 .unwrap();
14050 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14051 assert_eq!(intent["branch"], branch);
14052 assert_eq!(intent["start"], runner.state.base_commit);
14053 assert_eq!(
14054 intent["worktree"],
14055 runner
14056 .state
14057 .worktree_root()
14058 .join(format!("cand-{label}"))
14059 .to_string_lossy()
14060 .as_ref()
14061 );
14062 assert_eq!(intent["picks"], serde_json::json!([]));
14063 }
14064
14065 #[tokio::test]
14066 async fn invalid_pinned_sources_create_no_partial_worktrees() {
14067 ask_test_home();
14068 let tmp = tempfile::tempdir().unwrap();
14069 let repo = tmp.path().join("repo");
14070 std::fs::create_dir_all(&repo).unwrap();
14071 init_repo(&repo);
14072 let mut runner = runner_at(RunStatus::Prep);
14073 runner.state.repo = repo.clone();
14074 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14075 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14076 runner.state.sources.pick_commits.push(refs::Input {
14077 token: "missing".into(),
14078 sha: "0".repeat(40),
14079 provenance: refs::Provenance::ExplicitCli,
14080 });
14081 assert!(runner.prep().await.is_err());
14082 assert!(runner.state.candidates.is_empty());
14083 assert!(!runner.state.worktree_root().exists());
14084 }
14085
14086 #[tokio::test]
14087 async fn prose_and_legacy_inference_never_import_into_candidates() {
14088 ask_test_home();
14089 let tmp = tempfile::tempdir().unwrap();
14090 let repo = tmp.path().join("repo");
14091 std::fs::create_dir_all(&repo).unwrap();
14092 init_repo(&repo);
14093 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14094 git::git(&repo, &["checkout", "-b", "magi/history/A"])
14095 .await
14096 .unwrap();
14097 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14098 git::git(&repo, &["add", "."]).await.unwrap();
14099 git::git(&repo, &["commit", "-m", "historical implementation"])
14100 .await
14101 .unwrap();
14102 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14103 git::git(&repo, &["checkout", "main"]).await.unwrap();
14104 for instruction in [
14105 format!("Example: magi/history/A at {source}"),
14106 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14107 format!("Never import magi/history/A or {source}"),
14108 format!("Conversation answer: I mentioned {source} only as background"),
14109 ] {
14110 let mut runner = runner_at(RunStatus::Prep);
14111 runner.state.repo = repo.clone();
14112 runner.state.base_commit = base.clone();
14113 runner.state.instruction = instruction.clone();
14114 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14115 runner.state.config.blind.commit_msg_hook = false;
14116 runner.state.seeds = vec![refs::Seed {
14117 token: "magi/history/A".into(),
14118 sha: source.clone(),
14119 kind: refs::SeedKind::Unmerged,
14120 branch: true,
14121 detail: String::new(),
14122 }];
14123 runner.roles.implementers = vec![spec("one")];
14124 assert_eq!(seeded_instruction(&runner.state), instruction);
14125 let mut legacy = serde_json::to_value(&runner.state).unwrap();
14127 legacy.as_object_mut().unwrap().remove("sources");
14128 runner.state = serde_json::from_value(legacy).unwrap();
14129 assert_eq!(runner.state.sources, refs::Sources::default());
14130 runner.state.save().unwrap();
14131 runner.state = RunState::load(&runner.state.id).unwrap();
14132 runner.prep().await.unwrap();
14133 for candidate in &runner.state.candidates {
14134 assert_eq!(
14135 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14136 base
14137 );
14138 assert!(!candidate.worktree.join("unwanted").exists());
14139 }
14140 }
14141 }
14142
14143 #[tokio::test]
14144 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14145 ask_test_home();
14146 let tmp = tempfile::tempdir().unwrap();
14147 let repo = tmp.path().join("repo");
14148 std::fs::create_dir_all(&repo).unwrap();
14149 init_repo(&repo);
14150 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14151 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14152 std::fs::write(repo.join("referenced"), "required commit").unwrap();
14153 git::git(&repo, &["add", "."]).await.unwrap();
14154 git::git(&repo, &["commit", "-m", "required seed"])
14155 .await
14156 .unwrap();
14157 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14158 git::git(&repo, &["checkout", "main"]).await.unwrap();
14159 let mut runner = runner_at(RunStatus::Prep);
14160 runner.state.repo = repo.clone();
14161 runner.state.base_commit = base.clone();
14162 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14163 runner.state.config.graph.advise = false;
14164 runner.state.config.blind.commit_msg_hook = false;
14165 runner.state.sources =
14166 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14167 .await
14168 .unwrap();
14169 runner.roles.implementers = vec![spec("one"), spec("two")];
14170 runner.roles.judges = vec![spec("judge")];
14171 let label = blind::assign_labels(2, runner.state.seed)[0];
14172 let branch = runner.state.branch_for(label);
14173 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14174 git::worktree_add_branch(&repo, &worktree, &branch, &base)
14175 .await
14176 .unwrap();
14177 runner.state.candidates.push(Candidate {
14178 index: 0,
14179 label,
14180 agent: "one".into(),
14181 branch,
14182 worktree,
14183 summary: String::new(),
14184 stat: String::new(),
14185 files: 0,
14186 commits: 0,
14187 empty: false,
14188 failed: None,
14189 verified_noop: None,
14190 duration_ms: 0,
14191 folded: false,
14192 });
14193 runner.state.save().unwrap();
14194 runner.state = RunState::load(&runner.state.id).unwrap();
14196 assert_eq!(runner.state.prep_complete, Some(false));
14197 runner.prep().await.unwrap();
14198 assert_eq!(runner.state.prep_complete, Some(true));
14199 assert_eq!(runner.state.candidates.len(), 2);
14200 for candidate in &runner.state.candidates {
14201 assert_eq!(
14202 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14203 git::tree_of(&repo, &seed).await.unwrap()
14204 );
14205 assert_eq!(
14206 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14207 "required commit"
14208 );
14209 }
14210 assert!(runner.state.worktree_root().join("judge-1").exists());
14211 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14212 runner.prep().await.unwrap();
14213 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14214
14215 runner.state.prep_complete = Some(false);
14216 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14217 run: Some(format!(
14218 "test -f '{}'",
14219 tmp.path().join("dependency-ready").display()
14220 )),
14221 ..Default::default()
14222 }];
14223 assert!(runner.prep().await.is_err());
14224 assert!(runner.state.prep_failure.is_some());
14225 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14226 runner.state = RunState::load(&runner.state.id).unwrap();
14227 assert!(
14228 runner.prep().await.is_err(),
14229 "correcting the cause alone must not auto-retry"
14230 );
14231 let failed_at = runner.state.events.last().unwrap().at;
14232 assert!(
14233 runner.retry_preparation(failed_at).is_err(),
14234 "old approvals cannot unlock the hold"
14235 );
14236 let mut task = crate::queue::Task::new(
14237 "retry".into(),
14238 "retry setup".into(),
14239 repo.clone(),
14240 crate::queue::Source::Human,
14241 );
14242 task.start(runner.state.id.clone());
14243 task.hold_machine(Some("setup failed".into()));
14244 task.release_operator("cli", None);
14245 runner
14246 .retry_preparation(task.resumption.as_ref().unwrap().at)
14247 .unwrap();
14248 runner.prep().await.unwrap();
14249 assert!(runner.state.prep_failure.is_none());
14250 assert_eq!(runner.state.prep_complete, Some(true));
14251 assert_eq!(runner.state.candidates.len(), 2);
14252 assert!(runner.state.worktree_root().join("judge-1").exists());
14253 for candidate in &runner.state.candidates {
14254 assert_eq!(
14255 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14256 git::tree_of(&repo, &seed).await.unwrap()
14257 );
14258 }
14259 }
14260
14261 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14262 let mut heads = Vec::new();
14263 for candidate in candidates {
14264 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14265 }
14266 heads
14267 }
14268
14269 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14270 ask_test_home();
14271 let tmp = tempfile::tempdir().unwrap();
14272 let repo = tmp.path().join("repo");
14273 std::fs::create_dir(&repo).unwrap();
14274 init_repo(&repo);
14275 let mut runner = runner_at(RunStatus::Reviewing);
14276 runner.state.repo = repo.clone();
14277 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14278 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14279 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14280 let branch = format!("candidate-{label}");
14281 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14282 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14283 .await
14284 .unwrap();
14285 if label == 'B' {
14286 std::fs::write(worktree.join("change"), "winner").unwrap();
14287 git::rescue_commit(&worktree, "candidate change")
14288 .await
14289 .unwrap();
14290 }
14291 runner.state.candidates.push(Candidate {
14292 index,
14293 label,
14294 agent: "alpha".into(),
14295 branch,
14296 worktree,
14297 summary: "completed reply".into(),
14298 stat: String::new(),
14299 files: usize::from(label == 'B'),
14300 commits: usize::from(label == 'B'),
14301 empty: label == 'A',
14302 failed: None,
14303 verified_noop: None,
14304 duration_ms: 1,
14305 folded: false,
14306 });
14307 }
14308 runner.state.judge_skipped = true;
14309 runner.state.tally = Some(Tally {
14310 first_choice: BTreeMap::new(),
14311 borda: BTreeMap::new(),
14312 winner: 'B',
14313 rankings: 0,
14314 unanimous_initial: false,
14315 deliberated: false,
14316 changed_votes: 0,
14317 unanimous_final: false,
14318 tie_break: None,
14319 judges: 0,
14320 present: 0,
14321 quorum: 0,
14322 met_quorum: true,
14323 uncontested: Some("only B produced a change".into()),
14324 });
14325 runner.state.save().unwrap();
14326 (tmp, runner)
14327 }
14328
14329 #[tokio::test]
14330 #[cfg(unix)]
14331 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14332 for (solo, failed_checkpoint) in
14333 [(true, false), (false, false), (true, true), (false, true)]
14334 {
14335 let (_tmp, mut runner) = preservation_resume_fixture().await;
14336 if solo {
14337 runner.state.candidates.retain(|c| c.label == 'A');
14338 }
14339 runner.state.candidates[0].empty = false;
14340 runner.state.candidates[0].summary.clear();
14341 runner.state.tally = None;
14342 runner.state.judge_skipped = false;
14343 runner.state.status = RunStatus::Implementing;
14344 runner.state.config.disk.min_free_bytes = u64::MAX;
14345 let mut implementer = spec("alpha");
14346 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14347 runner.roles.implementers = vec![implementer];
14348 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14349 if failed_checkpoint {
14350 std::fs::create_dir_all(&patch).unwrap();
14351 }
14352 runner.state.save().unwrap();
14353 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14354 runner.state = RunState::load(&runner.state.id).unwrap();
14355 assert_eq!(runner.state.status, RunStatus::Blocked);
14356 assert!(runner.state.interrupted_implementations.contains(&'A'));
14357 assert_eq!(runner.state.candidates[0].commits, 0);
14358 assert!(!runner.state.candidates[0].empty);
14359 assert!(runner.state.candidates[0].failed.is_none());
14360 if failed_checkpoint {
14361 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14362 std::fs::remove_dir(patch).unwrap();
14363 }
14364 runner.state.resource_block = None;
14365 runner.state.config.disk.min_free_bytes = 0;
14366 runner.roles.implementers[0].command = vec![
14367 "sh".into(),
14368 "-c".into(),
14369 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14370 ];
14371 runner.implement().await.unwrap();
14372 assert!(runner.state.interrupted_implementations.is_empty());
14373 assert!(runner.state.candidate_preservation.is_empty());
14374 assert_eq!(runner.state.candidates[0].commits, 1);
14375 assert!(!runner.state.candidates[0].empty);
14376 assert!(
14377 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14378 .unwrap()
14379 .contains("retried empty interrupted seat")
14380 );
14381 if !solo {
14382 assert_eq!(runner.state.candidates[1].commits, 1);
14383 }
14384 }
14385 }
14386
14387 #[tokio::test]
14388 #[cfg(unix)]
14389 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14390 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14391 let (_tmp, mut runner) = preservation_resume_fixture().await;
14392 runner.state.candidates.retain(|c| c.label == 'A');
14393 runner.state.candidates[0].empty = false;
14394 runner.state.candidates[0].summary.clear();
14395 runner.state.tally = None;
14396 runner.state.judge_skipped = false;
14397 runner.state.status = RunStatus::Implementing;
14398 runner.state.config.disk.min_free_bytes = u64::MAX;
14401 let wt = runner.state.candidates[0].worktree.clone();
14402 std::fs::write(
14403 wt.join("interrupted-source"),
14404 "retain implementation work\n",
14405 )
14406 .unwrap();
14407 if committed {
14408 git::rescue_commit(&wt, "implementation checkpoint")
14409 .await
14410 .unwrap();
14411 }
14412 let lock = git::git(
14413 &wt,
14414 &[
14415 "rev-parse",
14416 "--path-format=absolute",
14417 "--git-path",
14418 "index.lock",
14419 ],
14420 )
14421 .await
14422 .unwrap();
14423 if block_rescue {
14424 std::fs::write(&lock, "fixture-owned lock").unwrap();
14425 }
14426 let mut implementer = spec("alpha");
14427 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14428 runner.roles.implementers = vec![implementer];
14429 runner.state.save().unwrap();
14430 let result = runner.implement().await;
14431 assert_eq!(result.is_err(), block_rescue);
14432 assert_eq!(
14433 runner.state.resource_block.as_ref().unwrap().kind,
14434 "disk pressure"
14435 );
14436 assert!(
14437 runner
14438 .state
14439 .events
14440 .iter()
14441 .any(|e| e.message.contains("agent execution interrupted"))
14442 );
14443 assert!(
14444 runner
14445 .state
14446 .dir()
14447 .join("artifacts/candidate-A-recovery/staged.patch")
14448 .exists()
14449 );
14450 if block_rescue {
14451 let saved = RunState::load(&runner.state.id).unwrap();
14452 assert!(saved.candidate_preservation.contains_key(&'A'));
14453 assert!(!saved.candidates[0].empty);
14454 assert_eq!(
14455 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14456 "retain implementation work\n"
14457 );
14458 std::fs::remove_file(lock).unwrap();
14459 runner.state = saved;
14460 } else {
14461 assert_eq!(runner.state.status, RunStatus::Blocked);
14462 assert_eq!(runner.state.candidates[0].commits, 1);
14463 }
14464 runner.roles.implementers.clear();
14467 runner.state.config.disk.min_free_bytes = 0;
14468 runner.state.resource_block = None;
14469 runner.implement().await.unwrap();
14470 assert!(runner.state.candidate_preservation.is_empty());
14471 assert_eq!(runner.state.candidates[0].commits, 1);
14472 assert!(!runner.state.candidates[0].empty);
14473 assert!(git::is_clean(&wt).await.unwrap());
14474 assert!(
14475 crate::run::read_artifact(&runner.state, "cand-A.patch")
14476 .unwrap()
14477 .contains("retain implementation work")
14478 );
14479 }
14480 }
14481
14482 #[tokio::test]
14483 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14484 let (_tmp, mut runner) = preservation_resume_fixture().await;
14485 runner.implement().await.unwrap();
14487 assert!(runner.state.candidate_preservation.is_empty());
14488 assert!(
14489 !runner
14490 .state
14491 .dir()
14492 .join("artifacts/candidate-A-recovery")
14493 .exists()
14494 );
14495 runner.fold_losers().await.unwrap();
14496 assert!(!runner.state.candidates[0].worktree.exists());
14497 for schema in [14, crate::run::SCHEMA] {
14498 runner.state.schema = schema;
14499 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14500 runner.state.save().unwrap();
14501 runner.state = RunState::load(&runner.state.id).unwrap();
14502 runner.implement().await.unwrap();
14503 assert!(runner.state.candidate_preservation.is_empty());
14504 assert!(runner.state.candidates[0].folded);
14505 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14506 }
14507 }
14508
14509 #[tokio::test]
14510 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14511 let (_tmp, mut runner) = preservation_resume_fixture().await;
14512 runner.state.candidates.retain(|c| c.label == 'A');
14513 runner.state.tally = None;
14514 runner.state.judge_skipped = false;
14515 let candidate = &mut runner.state.candidates[0];
14516 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14517 candidate.failed = Some("deployment validation unavailable".into());
14518 std::fs::write(
14519 candidate.worktree.join("repair"),
14520 "future launcher policy\n",
14521 )
14522 .unwrap();
14523 runner
14524 .state
14525 .candidate_preservation
14526 .insert('A', "awaiting persistence".into());
14527 runner.persist_candidates().await.unwrap();
14528 let candidate = &runner.state.candidates[0];
14529 assert!(!candidate.empty);
14530 assert_eq!(candidate.commits, 1);
14531 assert_eq!(candidate.files, 1);
14532 assert!(candidate.failed.is_none());
14533 assert!(candidate.summary.contains("EPERM"));
14534 assert!(
14535 crate::run::read_artifact(&runner.state, "cand-A.patch")
14536 .unwrap()
14537 .contains("future launcher policy")
14538 );
14539 assert!(runner.state.candidate_preservation.is_empty());
14540 }
14541
14542 #[tokio::test]
14543 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14544 let (_tmp, mut runner) = preservation_resume_fixture().await;
14545 let wt = runner.state.candidates[0].worktree.clone();
14546 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14547 runner.state.schema = 14;
14548 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14549 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14550 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14551 for _ in 0..2 {
14552 assert!(runner.implement().await.is_err());
14553 assert_eq!(runner.state.status, RunStatus::Blocked);
14554 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14555 assert!(
14556 runner.state.candidates[0].empty,
14557 "old verdict must remain unusable"
14558 );
14559 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14560 assert!(runner.state.judge_skipped);
14561 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14562 assert!(
14563 crate::run::read_artifact(&runner.state, "cand-A.patch")
14564 .unwrap()
14565 .contains("owned edits")
14566 );
14567 assert!(wt.exists());
14568 assert_eq!(
14569 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14570 .await
14571 .unwrap(),
14572 1
14573 );
14574 runner.state = RunState::load(&runner.state.id).unwrap();
14575 }
14576 }
14577
14578 #[tokio::test]
14579 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14580 let (_tmp, mut runner) = preservation_resume_fixture().await;
14581 let wt = runner.state.candidates[0].worktree.clone();
14582 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14583 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14584 let index_lock = PathBuf::from(
14585 git::git(
14586 &wt,
14587 &[
14588 "rev-parse",
14589 "--path-format=absolute",
14590 "--git-path",
14591 "index.lock",
14592 ],
14593 )
14594 .await
14595 .unwrap(),
14596 );
14597 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14598 runner.state.schema = 14;
14599 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14600 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14601 assert!(runner.implement().await.is_err());
14602 assert!(runner.after_implement().is_err());
14603 assert!(runner.fold_losers().await.is_err());
14604 assert_eq!(
14605 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14606 "failed seat edits"
14607 );
14608 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14609 runner.state = RunState::load(&runner.state.id).unwrap();
14610 std::fs::remove_file(index_lock).unwrap();
14611 assert!(runner.implement().await.is_err());
14613 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14614 assert!(runner.state.candidates[0].empty);
14615 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14616 assert!(
14617 crate::run::read_artifact(&runner.state, "cand-A.patch")
14618 .unwrap()
14619 .contains("failed seat edits")
14620 );
14621 assert!(wt.exists());
14622 }
14623
14624 #[tokio::test]
14625 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14626 let (_tmp, mut runner) = preservation_resume_fixture().await;
14627 let repo = runner.state.repo.clone();
14628 let common = git::git(
14629 &repo,
14630 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14631 )
14632 .await
14633 .unwrap();
14634 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14635 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14636 assert!(runner.fold_losers().await.is_err());
14637 assert!(!runner.state.candidates[0].worktree.exists());
14638 assert!(!runner.state.candidates[0].folded);
14639 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14640 runner.state = RunState::load(&runner.state.id).unwrap();
14641 std::fs::remove_file(branch_lock).unwrap();
14642 let judge = runner.state.worktree_root().join("judge-1");
14643 git::worktree_add_detached(&repo, &judge, "main")
14644 .await
14645 .unwrap();
14646 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14647 runner.roles.judges = vec![spec("judge")];
14648 assert!(runner.fold_losers().await.is_err());
14649 runner.state = RunState::load(&runner.state.id).unwrap();
14650 assert!(
14651 runner.state.candidates[0].folded,
14652 "candidate progress saved before judge failure"
14653 );
14654 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14655 assert!(judge.join("uncommitted").exists());
14656 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14657 runner.fold_losers().await.unwrap();
14658 assert!(!judge.exists());
14659 runner.fold_losers().await.unwrap();
14660 }
14661
14662 #[tokio::test]
14663 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14664 let (_tmp, mut runner) = preservation_resume_fixture().await;
14665 runner.state.candidates.retain(|c| c.label == 'B');
14666 runner.state.tally = None;
14667 runner.state.judge_skipped = false;
14668 runner.state.prep_complete = Some(true);
14669 runner.state.config.graph.advise = false;
14670 runner.state.config.graph.selected_implementer = Some("alpha".into());
14671 let wt = runner.state.candidates[0].worktree.clone();
14672 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14673 runner
14674 .state
14675 .candidate_preservation
14676 .insert('B', "awaiting persistence".into());
14677 let index_lock = git::git(
14678 &wt,
14679 &[
14680 "rev-parse",
14681 "--path-format=absolute",
14682 "--git-path",
14683 "index.lock",
14684 ],
14685 )
14686 .await
14687 .unwrap();
14688 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14689 runner.state.save().unwrap();
14690 assert!(runner.execute_graph().await.is_err());
14691 assert_eq!(runner.state.status, RunStatus::Blocked);
14692 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14693 std::fs::remove_file(index_lock).unwrap();
14694 for _ in 0..2 {
14697 runner.state = RunState::load(&runner.state.id).unwrap();
14698 runner.pause.park();
14699 runner.execute_graph().await.unwrap();
14700 assert_eq!(runner.state.status, RunStatus::Implementing);
14701 assert!(runner.state.candidate_preservation.is_empty());
14702 assert_eq!(runner.state.candidates[0].commits, 2);
14703 assert!(runner.state.tally.is_none());
14704 }
14705 runner.implement().await.unwrap();
14706 assert_eq!(runner.state.status, RunStatus::Judging);
14707 runner.state.status = RunStatus::Blocked;
14709 runner.state.candidates[0].failed = Some("implementation failed".into());
14710 runner.state.save().unwrap();
14711 let error = runner.execute_graph().await.unwrap_err();
14712 assert!(
14713 error
14714 .to_string()
14715 .contains("selected implementation is held")
14716 );
14717 }
14718
14719 #[tokio::test]
14720 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14721 let (_tmp, mut runner) = preservation_resume_fixture().await;
14722 let repo = runner.state.repo.clone();
14723 let candidate = runner.state.candidates[0].clone();
14724 let registered = git::worktree_holding(&repo, &candidate.branch)
14727 .await
14728 .unwrap()
14729 .unwrap();
14730 assert_eq!(
14731 registered.canonicalize().unwrap(),
14732 candidate.worktree.canonicalize().unwrap()
14733 );
14734 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14735 assert_eq!(
14736 git::worktree_holding(&repo, &candidate.branch)
14737 .await
14738 .unwrap(),
14739 Some(registered)
14740 );
14741 assert!(
14742 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14743 "the stale registration initially keeps the branch checked out"
14744 );
14745 runner.fold_losers().await.unwrap();
14746 assert!(runner.state.candidates[0].folded);
14747 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14748 assert_eq!(
14749 git::worktree_holding(&repo, &candidate.branch)
14750 .await
14751 .unwrap(),
14752 None
14753 );
14754 runner.state = RunState::load(&runner.state.id).unwrap();
14755 runner.fold_losers().await.unwrap();
14756 }
14757
14758 #[tokio::test]
14759 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14760 let (_tmp, mut runner) = preservation_resume_fixture().await;
14761 let repo = runner.state.repo.clone();
14762 let candidate = runner.state.candidates[0].clone();
14763 let winner = runner.state.candidates[1].clone();
14764 let registered = git::worktree_holding(&repo, &candidate.branch)
14765 .await
14766 .unwrap()
14767 .unwrap();
14768 assert_eq!(
14769 registered.canonicalize().unwrap(),
14770 candidate.worktree.canonicalize().unwrap()
14771 );
14772 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14773 assert_eq!(
14774 git::worktree_holding(&repo, &candidate.branch)
14775 .await
14776 .unwrap(),
14777 Some(registered)
14778 );
14779 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14780
14781 for _ in 0..2 {
14782 fold_run(&mut runner.state, false, &crate::run::home())
14783 .await
14784 .unwrap();
14785 runner.state = RunState::load(&runner.state.id).unwrap();
14786 assert!(runner.state.candidates[0].folded);
14787 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14788 assert_eq!(
14789 git::worktree_holding(&repo, &candidate.branch)
14790 .await
14791 .unwrap(),
14792 None
14793 );
14794 assert!(!runner.state.candidates[1].folded);
14795 assert!(winner.worktree.exists());
14796 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14797 }
14798 }
14799
14800 #[tokio::test]
14801 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14802 ask_test_home();
14803 let tmp = tempfile::tempdir().unwrap();
14804 let repo = tmp.path().join("repo");
14805 std::fs::create_dir(&repo).unwrap();
14806 init_repo(&repo);
14807 let mut runner = runner_at(RunStatus::Implementing);
14808 runner.state.repo = repo.clone();
14809 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14810 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14811 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14812 let branch = format!("candidate-{label}");
14813 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14814 git::worktree_add_branch(&repo, &wt, &branch, "main")
14815 .await
14816 .unwrap();
14817 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14818 git::git(&wt, &["add", "README.md"]).await.unwrap();
14819 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14820 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14821 runner.state.candidates.push(Candidate {
14822 index,
14823 label,
14824 agent: "alpha".to_owned(),
14825 branch,
14826 worktree: wt,
14827 summary: "completed implementation".to_owned(),
14828 stat: String::new(),
14829 files: 0,
14830 commits: 0,
14831 empty: false,
14832 failed: None,
14833 verified_noop: None,
14834 duration_ms: 42,
14835 folded: false,
14836 });
14837 runner
14838 .state
14839 .candidate_preservation
14840 .insert(label, "awaiting persistence".to_owned());
14841 }
14842 let wt = runner.state.candidates[0].worktree.clone();
14843 let index_lock = git::git(
14844 &wt,
14845 &[
14846 "rev-parse",
14847 "--path-format=absolute",
14848 "--git-path",
14849 "index.lock",
14850 ],
14851 )
14852 .await
14853 .unwrap();
14854 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14855 runner.state.save().unwrap();
14856 assert!(runner.persist_candidates().await.is_err());
14857 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14858 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14859 assert_eq!(
14860 runner.state.candidates[1].commits, 1,
14861 "other results are still collected"
14862 );
14863 assert!(
14864 !runner.state.candidates[0].empty,
14865 "failed rescue is not no change"
14866 );
14867 assert_eq!(
14868 runner.state.candidates[0].summary,
14869 "completed implementation"
14870 );
14871 assert!(runner.after_implement().is_err());
14872 let home = tmp.path().join("fold-home");
14873 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14874 assert!(wt.join("new-file").exists());
14875 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14876 assert!(recovery.join("staged.patch").exists());
14877 assert!(recovery.join("unstaged.patch").exists());
14878 assert_eq!(
14879 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14880 "untracked edits\n"
14881 );
14882 runner.state.begin_pass();
14883 runner.state.end_pass_parked(ParkCause::Upgrade {
14884 from: "1.0".into(),
14885 to: Some("1.1".into()),
14886 });
14887 runner.state.save().unwrap();
14888 runner.state = RunState::load(&runner.state.id).unwrap();
14889 assert!(matches!(
14890 runner.state.passes.last().unwrap().end,
14891 Some(crate::run::PassEnd::Parked {
14892 cause: ParkCause::Upgrade { .. }
14893 })
14894 ));
14895 std::fs::remove_file(index_lock).unwrap();
14896 runner.implement().await.unwrap();
14898 assert!(runner.state.candidate_preservation.is_empty());
14899 assert!(
14900 runner
14901 .state
14902 .candidates
14903 .iter()
14904 .all(|c| c.commits == 1 && !c.empty)
14905 );
14906 assert!(git::is_clean(&wt).await.unwrap());
14907 assert!(
14908 crate::run::read_artifact(&runner.state, "cand-A.patch")
14909 .unwrap()
14910 .contains("later tracked edits")
14911 );
14912 }
14913
14914 #[tokio::test]
14915 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14916 ask_test_home();
14917 let tmp = tempfile::tempdir().unwrap();
14918 let repo = tmp.path().join("repo");
14919 std::fs::create_dir(&repo).unwrap();
14920 init_repo(&repo);
14921 let wt = tmp.path().join("candidate");
14922 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14923 .await
14924 .unwrap();
14925 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14926 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14927 assert!(wt.join("uncommitted").exists());
14928 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14929 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14930 let mut runner = runner_at(RunStatus::Implementing);
14931 runner.state.repo = repo.clone();
14932 runner.state.base_commit = "missing-base".to_owned();
14933 runner.state.candidates.push(Candidate {
14934 index: 0,
14935 label: 'A',
14936 agent: "alpha".into(),
14937 branch: "candidate".into(),
14938 worktree: wt.clone(),
14939 summary: "completed".into(),
14940 stat: String::new(),
14941 files: 0,
14942 commits: 0,
14943 empty: false,
14944 failed: None,
14945 verified_noop: None,
14946 duration_ms: 1,
14947 folded: false,
14948 });
14949 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14950 assert!(
14951 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14952 .await
14953 .is_err()
14954 );
14955 assert!(!runner.state.candidates[0].folded);
14956 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14957 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14958 runner
14959 .state
14960 .candidate_preservation
14961 .insert('A', "pending".into());
14962 assert!(runner.persist_candidates().await.is_err());
14963 assert!(!runner.state.candidates[0].empty);
14964 assert!(!runner.state.candidates[0].folded);
14965 assert!(wt.exists());
14966 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14967 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14968 std::fs::create_dir_all(&patch_path).unwrap();
14969 assert!(
14970 runner.persist_candidates().await.is_err(),
14971 "artifact failure stays pending"
14972 );
14973 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14974 assert!(!runner.state.candidates[0].empty);
14975 std::fs::remove_dir(patch_path).unwrap();
14976 runner.persist_candidates().await.unwrap();
14977 assert!(
14978 runner.state.candidates[0].empty,
14979 "only successful inspection confirms no change"
14980 );
14981 }
14982
14983 fn init_repo(dir: &Path) {
14984 let run = |args: &[&str]| {
14985 let out = std::process::Command::new("git")
14986 .args(args)
14987 .current_dir(dir)
14988 .quiet()
14989 .output()
14990 .expect("spawn git");
14991 assert!(
14992 out.status.success(),
14993 "git {args:?} failed: {}",
14994 String::from_utf8_lossy(&out.stderr)
14995 );
14996 };
14997 run(&["init", "-b", "main"]);
14998 run(&["config", "user.name", "magi test"]);
14999 run(&["config", "user.email", "magi@example.com"]);
15000 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15001 run(&["add", "-A"]);
15002 run(&["commit", "-m", "init"]);
15003 }
15004
15005 fn ask_test_home() {
15013 crate::run::pin_test_home();
15014 }
15015
15016 #[cfg(unix)]
15017 async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15018 ask_test_home();
15019 let tmp = tempfile::tempdir().unwrap();
15020 let repo = tmp.path().join("repo");
15021 std::fs::create_dir(&repo).unwrap();
15022 init_repo(&repo);
15023 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15024 let origin = tmp.path().join("origin.git");
15025 git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15026 .await
15027 .unwrap();
15028 git::git(
15029 &repo,
15030 &["remote", "add", "origin", origin.to_str().unwrap()],
15031 )
15032 .await
15033 .unwrap();
15034 git::git(&repo, &["checkout", "-b", "fix/retained"])
15035 .await
15036 .unwrap();
15037 std::fs::write(repo.join("retained"), "retained change").unwrap();
15038 git::git(&repo, &["add", "."]).await.unwrap();
15039 git::git(&repo, &["commit", "-m", "retained implementation"])
15040 .await
15041 .unwrap();
15042 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15043 git::git(&repo, &["checkout", "main"]).await.unwrap();
15044 std::fs::write(repo.join("base-change"), "new base").unwrap();
15045 git::git(&repo, &["add", "."]).await.unwrap();
15046 git::git(&repo, &["commit", "-m", "base moved"])
15047 .await
15048 .unwrap();
15049 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15050 git::git(&repo, &["checkout", "fix/retained"])
15051 .await
15052 .unwrap();
15053 let mut runner = runner_at(RunStatus::Voting);
15054 runner.state.repo = repo.clone();
15055 runner.state.base_commit = base;
15056 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15057 runner.state.config.disk.min_free_bytes = 0;
15058 runner.state.config.graph.review_rounds = 1;
15059 runner.state.config.verify.e2e = vec!["true".into()];
15060 runner.state.config.verify.gate = vec!["true".into()];
15061 runner.state.candidates = vec![Candidate {
15062 index: 0,
15063 label: 'A',
15064 agent: "fixture".into(),
15065 branch: "fix/retained".into(),
15066 worktree: repo.clone(),
15067 summary: String::new(),
15068 stat: String::new(),
15069 files: 1,
15070 commits: 1,
15071 empty: false,
15072 failed: None,
15073 verified_noop: None,
15074 duration_ms: 0,
15075 folded: false,
15076 }];
15077 runner.state.tally = Some(Tally {
15078 first_choice: BTreeMap::new(),
15079 borda: BTreeMap::new(),
15080 winner: 'A',
15081 rankings: 0,
15082 unanimous_initial: true,
15083 deliberated: false,
15084 changed_votes: 0,
15085 unanimous_final: true,
15086 tie_break: None,
15087 judges: 0,
15088 present: 0,
15089 quorum: 0,
15090 met_quorum: true,
15091 uncontested: Some("review-only".into()),
15092 });
15093 let mut task = crate::queue::Task::new(
15094 "fixture".into(),
15095 "fixture".into(),
15096 repo,
15097 crate::queue::Source::Human,
15098 );
15099 task.link_run(&runner.state.id);
15100 crate::queue::Queue::open().put(&mut task).unwrap();
15101 runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15102 runner.state.base_sync = Some(BaseSync {
15103 tip: String::new(),
15104 behind: 1,
15105 attempts: BASE_SYNC_ROUNDS,
15106 conflict: None,
15107 already_in: None,
15108 });
15109 runner.sync_to_base().await.unwrap();
15110 assert_eq!(runner.state.status, RunStatus::Blocked);
15111 (tmp, runner, head, task.id)
15112 }
15113
15114 #[cfg(unix)]
15115 #[tokio::test]
15116 async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15117 const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15118 if std::env::var_os(CHILD).is_none() {
15119 let home = tempfile::tempdir().unwrap();
15120 let output = std::process::Command::new(std::env::current_exe().unwrap())
15121 .args([
15122 "--exact",
15123 "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15124 "--nocapture",
15125 ])
15126 .env(CHILD, "1")
15127 .env("MAGI_HOME", home.path())
15128 .env("MAGI_WORKER_HOME", home.path())
15129 .env("MAGI_WORKER_ID", "abcdef")
15130 .output()
15131 .unwrap();
15132 assert!(
15133 output.status.success()
15134 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15135 "{}\n{}",
15136 String::from_utf8_lossy(&output.stdout),
15137 String::from_utf8_lossy(&output.stderr)
15138 );
15139 return;
15140 }
15141 crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15142 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15143 let home = crate::run::home();
15144 let queue = crate::queue::Queue::open();
15145 let mut owner = queue.get(&task).unwrap();
15146 owner.release_operator("test-release", None);
15147 owner.start(runner.state.id.clone());
15148 queue.put(&mut owner).unwrap();
15149 runner.state.driver_exited = true;
15150 runner.state.task_instruction = Some(owner.instruction.clone());
15151 runner.state.save().unwrap();
15152 let _claim = queue.claim(&task).unwrap();
15153 let reserve = |id: &str| {
15154 let directory = home.join("workers/jobs").join(id);
15155 std::fs::create_dir_all(&directory).unwrap();
15156 let job = crate::worker::Job {
15157 id: id.into(),
15158 sequence: 1,
15159 kind: "task".into(),
15160 key: format!("task:{task}"),
15161 payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15162 generation: "fixture-generation".into(),
15163 version: None,
15164 status: crate::worker::Status::Running,
15165 result: None,
15166 pid: Some(std::process::id()),
15167 created_at: 0,
15168 finished_at: None,
15169 };
15170 std::fs::write(
15171 directory.join("job.json"),
15172 serde_json::to_vec(&job).unwrap(),
15173 )
15174 .unwrap();
15175 crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15176 };
15177 let _own = reserve("abcdef");
15178 let other = reserve("fedcba");
15179 assert!(
15180 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15181 .await
15182 .is_err()
15183 );
15184 assert!(
15185 RunState::load(&runner.state.id)
15186 .unwrap()
15187 .base_sync_recovery
15188 .authorizations
15189 .is_empty()
15190 );
15191 drop(other);
15192 assert!(crate::daemon::is_working_on(
15193 &home,
15194 &runner.state.id,
15195 Timestamp::now()
15196 ));
15197 assert!(
15198 !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15199 );
15200 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15201 .await
15202 .unwrap();
15203 assert!(
15204 RunState::load(&runner.state.id)
15205 .unwrap()
15206 .base_recovery_pending()
15207 );
15208 }
15209
15210 #[cfg(unix)]
15211 #[tokio::test]
15212 async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15213 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15214 let home = crate::run::home();
15215 runner.state.driver_pid = Some(std::process::id());
15216 runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15217 runner.state.driver_exited = false;
15218 runner.state.begin_pass();
15219 runner.state.save().unwrap();
15220 let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15221 driver.record(&runner.state).unwrap();
15222 let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15223 drop(driver);
15225 std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15226 assert_eq!(
15227 runner.state.liveness_under(&home, false),
15228 crate::run::Liveness::Dead
15229 );
15230 let queue = crate::queue::Queue::open();
15231 let _claim = queue.claim(&task).unwrap();
15232 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15233 .await
15234 .unwrap();
15235 let disk = RunState::load(&runner.state.id).unwrap();
15236 assert!(!disk.driver_exited);
15237 assert!(disk.base_recovery_pending());
15238 }
15239
15240 #[cfg(unix)]
15241 #[tokio::test]
15242 async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15243 const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15244 if std::env::var_os(CHILD).is_none() {
15245 use std::os::unix::fs::PermissionsExt;
15246 let tmp = tempfile::tempdir().unwrap();
15247 let gh = tmp.path().join("gh");
15248 std::fs::write(&gh, r#"#!/bin/sh
15249case "$1 $2" in
15250 'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15251 'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15252 *) exit 2 ;;
15253esac
15254"#).unwrap();
15255 std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15256 let mut paths = vec![tmp.path().to_path_buf()];
15257 paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15258 let output = std::process::Command::new(std::env::current_exe().unwrap())
15259 .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15260 .env(CHILD, "1")
15261 .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15262 .env("PATH", std::env::join_paths(paths).unwrap())
15263 .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15264 .output().unwrap();
15265 assert!(
15266 output.status.success()
15267 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15268 "{}\n{}",
15269 String::from_utf8_lossy(&output.stdout),
15270 String::from_utf8_lossy(&output.stderr)
15271 );
15272 return;
15273 }
15274 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15275 let repo = runner.state.repo.clone();
15276 git::git(&repo, &["checkout", "main"]).await.unwrap();
15277 git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15278 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15279 git::git(&repo, &["checkout", "fix/retained"])
15280 .await
15281 .unwrap();
15282 runner.state.pr = Some(crate::run::PrRecord {
15283 url: "https://example.test/project/pull/1".into(),
15284 number: 1,
15285 state: "open".into(),
15286 checks: "unknown".into(),
15287 round: 0,
15288 rounds: 1,
15289 red_at_merge: Vec::new(),
15290 });
15291 runner.sync_to_base().await.unwrap();
15292 assert_eq!(runner.state.status, RunStatus::Blocked);
15293 assert_eq!(
15294 runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15295 crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15296 );
15297 assert!(runner.state.base_sync_closure_pending());
15298 assert!(crate::handover::can_resume(&runner.state));
15299 let queue = crate::queue::Queue::open();
15300 assert!(!crate::base_sync_recovery::held_task(
15301 &queue,
15302 &queue.get(&task).unwrap()
15303 ));
15304 assert!(runner.state.authorize_base_sync(&task).is_err());
15305 runner.state.base_sync_recovery.stop = None;
15307 runner.state.status = RunStatus::Voting;
15308 runner.state.gate_ran = true;
15309 runner.state.config.graph.review_rounds = 0;
15310 runner.state.save().unwrap();
15311 assert!(crate::handover::can_resume(&runner.state));
15312 runner.state = RunState::load(&runner.state.id).unwrap();
15313 runner.execute_graph().await.unwrap();
15314 assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15315 assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15316 assert_eq!(
15317 runner.state.base_sync.as_ref().unwrap().attempts,
15318 BASE_SYNC_ROUNDS
15319 );
15320 assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15321 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15322 assert!(runner.state.reviews.is_empty());
15323 assert!(runner.state.gate.is_empty());
15324 }
15325
15326 #[cfg(unix)]
15327 #[tokio::test]
15328 async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15329 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15330 runner.state.driver_exited = true;
15331 runner.state.save().unwrap();
15332 let home = crate::run::home();
15333 let queue = crate::queue::Queue::open();
15334 let _claim = queue.claim(&task).unwrap();
15335 let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15336 assert!(
15337 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15338 .await
15339 .is_err()
15340 );
15341 drop(lease);
15342 std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15343 assert!(
15344 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15345 .await
15346 .is_err()
15347 );
15348 std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15349 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15350 .await
15351 .unwrap();
15352 let disk = RunState::load(&runner.state.id).unwrap();
15353 assert!(disk.base_recovery_pending());
15354 assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15355 assert!(
15356 Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15357 .await
15358 .is_err()
15359 );
15360 assert_eq!(
15361 RunState::load(&runner.state.id)
15362 .unwrap()
15363 .base_sync_recovery
15364 .authorizations
15365 .len(),
15366 1
15367 );
15368 let mut owner = queue.get(&task).unwrap();
15369 owner.runs.push("20260101-000000-ffff".into());
15370 queue.put(&mut owner).unwrap();
15371 let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15372 .await
15373 .unwrap_err();
15374 assert!(error.to_string().contains("run task ownership changed"));
15375 }
15376
15377 #[cfg(unix)]
15378 #[tokio::test]
15379 async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15380 let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15381 let mut legacy = serde_json::to_value(&runner.state).unwrap();
15382 legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15383 legacy["status"] = serde_json::json!("voting");
15384 runner.state = serde_json::from_value(legacy).unwrap();
15385 runner.state.save().unwrap();
15386 for _ in 0..3 {
15387 runner.execute_graph().await.unwrap();
15388 assert_eq!(runner.state.status, RunStatus::Blocked);
15389 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15390 assert!(!crate::handover::can_resume(&runner.state));
15391 assert_eq!(
15392 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15393 head
15394 );
15395 assert!(runner.state.reviews.is_empty());
15396 assert!(!runner.state.gate_ran);
15397 }
15398 let notice = crate::notices::run_ended(&runner.state).unwrap();
15399 let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15400 runner.state.begin_pass();
15401 runner.state.status = RunStatus::Voting;
15402 runner.deliberate().await.unwrap();
15403 runner.vote().await.unwrap();
15404 runner.finish_after_tally().await.unwrap();
15405 assert_eq!(runner.state.status, RunStatus::Blocked);
15406 assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15407 assert_eq!(
15408 crate::notices::run_ended(&runner.state).unwrap().since,
15409 notice.since
15410 );
15411 }
15412
15413 #[cfg(unix)]
15414 #[tokio::test]
15415 async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15416 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15417 for detail in [
15418 "fix/retained conflicts with origin/main and did not rebase: conflict",
15419 "rebased fix/retained locally but origin refused the push",
15420 "unknown failure",
15421 ] {
15422 runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15423 runner.state.record_base_sync_stop(&head);
15424 assert!(runner.state.authorize_base_sync(&task).is_err());
15425 runner.state.status = RunStatus::Voting;
15426 runner.execute_graph().await.unwrap();
15427 assert_eq!(runner.state.status, RunStatus::Blocked);
15428 assert!(!crate::handover::can_resume(&runner.state));
15429 assert_eq!(
15430 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15431 head
15432 );
15433 }
15434 }
15435
15436 #[cfg(unix)]
15437 #[tokio::test]
15438 async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15439 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15440 runner.state.authorize_base_sync(&task).unwrap();
15441 assert!(runner.state.consume_base_recovery(&head));
15442 runner.state.save().unwrap();
15443 runner.state = RunState::load(&runner.state.id).unwrap();
15444 runner.execute_graph().await.unwrap();
15445 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15446 assert_eq!(runner.state.status, RunStatus::Blocked);
15447 assert!(!runner.state.consume_base_recovery(&head));
15448 assert!(!crate::handover::can_resume(&runner.state));
15449 runner.state.authorize_base_sync(&task).unwrap();
15450 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15451 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15452 assert_eq!(
15453 git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15454 head
15455 );
15456 }
15457
15458 #[cfg(unix)]
15459 #[tokio::test]
15460 async fn base_recovery_success_requires_new_review_e2e_and_gate() {
15461 let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15462 let mut old_review = review_round(true, 0, 1, 1, false, true);
15463 old_review.head = head.clone();
15464 old_review.verified_head = Some(head.clone());
15465 runner.state.reviews = vec![old_review];
15466 runner.state.gate = vec![CommandOutcome {
15467 command: "old gate".into(),
15468 code: None,
15469 output_tail: "timed out after 2400s".into(),
15470 duration_ms: 2_400_000,
15471 resource_blocked: false,
15472 }];
15473 runner.state.gate_ran = true;
15474 runner.state.gate_head = Some(head.clone());
15475 runner.state.land_approval = Some(crate::run::LandApproval {
15476 question: "old-head-approval".into(),
15477 head: head.clone(),
15478 });
15479 runner.state.authorize_base_sync(&task).unwrap();
15480 runner.sync_to_base().await.unwrap();
15481 assert!(!runner.state.base_sync_blocked());
15482 let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15483 assert_ne!(head, fresh);
15484 assert!(runner.state.reviews.is_empty());
15485 assert!(!runner.state.gate_ran);
15486 let saved = &runner.state.base_sync_recovery.authorizations[0];
15487 assert_eq!(saved.reviews[0].head, head);
15488 assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
15489 assert!(saved.gate_ran);
15490 assert!(!saved.gate[0].ok());
15491 runner.review_loop().await.unwrap();
15492 runner.gate().await.unwrap();
15493 assert_eq!(runner.state.reviews[0].head, fresh);
15494 assert_eq!(
15495 runner.state.reviews[0].verified_head.as_deref(),
15496 Some(fresh.as_str())
15497 );
15498 assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
15499 assert!(runner.state.gate_ran);
15500 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15501 assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
15502 assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
15503 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15504 assert_eq!(
15505 std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
15506 "retained change"
15507 );
15508 }
15509
15510 #[cfg(unix)]
15511 #[tokio::test]
15512 async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
15513 let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15514 let queue = crate::queue::Queue::open();
15515 let mut owner = queue.get(&task).unwrap();
15516 owner.release_operator("test-resume", None);
15517 queue.put(&mut owner).unwrap();
15518 let generation = owner.resumption.as_ref().unwrap().generation;
15519 runner.state.task_instruction = Some(owner.instruction.clone());
15520 runner.state.driver_exited = true;
15521 runner.state.save().unwrap();
15522 let home = crate::run::home();
15523 let _claim = queue.claim(&task).unwrap();
15524 owner.resumption.as_mut().unwrap().at =
15525 runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
15526 queue.put(&mut owner).unwrap();
15527 assert!(
15528 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15529 .await
15530 .is_err(),
15531 "an approval from before the stop grants nothing"
15532 );
15533 owner.resumption.as_mut().unwrap().at = Timestamp::now();
15534 queue.put(&mut owner).unwrap();
15535 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15536 .await
15537 .unwrap();
15538 assert!(
15539 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15540 .await
15541 .is_err()
15542 );
15543 runner.state = RunState::load(&runner.state.id).unwrap();
15544 assert_eq!(
15545 runner.state.base_sync_recovery.authorizations[0].generation,
15546 Some(generation)
15547 );
15548 runner.sync_to_base().await.unwrap();
15549 let repo = runner.state.repo.clone();
15550 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15551 git::git(&repo, &["checkout", "main"]).await.unwrap();
15552 std::fs::write(repo.join("base-change"), "base moved again").unwrap();
15553 git::git(&repo, &["add", "."]).await.unwrap();
15554 git::git(&repo, &["commit", "-m", "move base again"])
15555 .await
15556 .unwrap();
15557 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15558 git::git(&repo, &["checkout", "fix/retained"])
15559 .await
15560 .unwrap();
15561 runner.finish_after_tally().await.unwrap();
15562 assert_eq!(runner.state.status, RunStatus::Blocked);
15563 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15564 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15565 assert!(!runner.state.base_recovery_pending());
15566 assert!(!crate::handover::can_resume(&runner.state));
15567 assert!(!runner.state.gate_ran);
15568 owner.resumption.as_mut().unwrap().at = Timestamp::now();
15570 queue.put(&mut owner).unwrap();
15571 assert!(
15572 Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15573 .await
15574 .is_err()
15575 );
15576 runner.state = RunState::load(&runner.state.id).unwrap();
15577 runner.sync_to_base().await.unwrap();
15578 assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15579 assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
15580 assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15581 }
15582
15583 #[cfg(unix)]
15584 #[tokio::test]
15585 async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
15586 let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
15587 runner.state.base_sync = None;
15588 runner.state.status = RunStatus::Blocked;
15589 let mut second = runner.state.candidates[0].clone();
15590 second.label = 'B';
15591 runner.state.candidates.push(second);
15592 assert!(runner.state.votes.is_empty());
15593 assert!(runner.state.deliberation.is_empty());
15594 runner.deliberate().await.unwrap();
15595 runner.vote().await.unwrap();
15596 assert_eq!(runner.state.status, RunStatus::Blocked);
15597 assert!(runner.state.votes.is_empty());
15598 assert!(runner.state.deliberation.is_empty());
15599 }
15600
15601 fn runner_at(status: RunStatus) -> Runner {
15604 let mut state = RunState::new(
15605 PathBuf::from("/nonexistent/repo"),
15606 "main".to_owned(),
15607 "deadbeef".to_owned(),
15608 "task".to_owned(),
15609 Config::default(),
15610 );
15611 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
15614 state.id = format!(
15615 "20000101-000000-{:04x}",
15616 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
15617 );
15618 state.status = status;
15619 Runner {
15620 state,
15621 roles: ResolvedRoles {
15622 implementers: Vec::new(),
15623 judges: Vec::new(),
15624 reviewers: Vec::new(),
15625 fixer: None,
15626 conductor: conductor(),
15627 implementer_roster: Vec::new(),
15628 judge_roster: Vec::new(),
15629 reviewer_roster: Vec::new(),
15630 },
15631 sem: Arc::new(Semaphore::new(1)),
15632 pause: Pause::new(),
15633 interrupt: Pause::new(),
15634 }
15635 }
15636
15637 #[tokio::test]
15638 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
15639 crate::run::pin_test_home();
15640 let mut runner = runner_at(RunStatus::Gating);
15641 runner.state.save().unwrap();
15642 let id = runner.state.id.clone();
15643 let passes = runner.state.passes.len();
15644 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15645 let mut claiming = Box::pin(runner.claim_driver());
15646 std::future::poll_fn(|cx| {
15647 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
15648 std::task::Poll::Ready(())
15649 })
15650 .await;
15651 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
15652 drop(owner);
15653 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
15654 .await
15655 .unwrap()
15656 .unwrap();
15657 assert_eq!(execution.runner.state.passes.len(), passes + 1);
15658 assert!(execution.runner.state.resource_block.is_none());
15659 assert_eq!(
15660 execution.runner.state.liveness(false),
15661 crate::run::Liveness::Live
15662 );
15663 drop(execution);
15664 assert!(RunState::load(&id).unwrap().driver_exited);
15665 }
15666
15667 #[tokio::test]
15668 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
15669 crate::run::pin_test_home();
15670 let mut runner = runner_at(RunStatus::Gating);
15671 runner.state.save().unwrap();
15672 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
15675 assert!(runner.claim_driver().await.is_err());
15676 assert!(runner.state.driver_exited);
15677 assert_eq!(runner.state.driver_pid, None);
15678 let disk = RunState::load(&runner.state.id).unwrap();
15679 assert!(disk.driver_exited);
15680 assert_eq!(disk.driver_pid, None);
15681 assert_eq!(disk.status, RunStatus::Gating);
15682 }
15683
15684 #[tokio::test]
15685 async fn an_early_graph_error_releases_its_driver() {
15686 crate::run::pin_test_home();
15687 let mut runner = runner_at(RunStatus::Blocked);
15688 runner.state.config.graph.selected_implementer = Some("selected".into());
15689 runner.state.save().unwrap();
15690 assert!(runner.execute().await.is_err());
15691 let disk = RunState::load(&runner.state.id).unwrap();
15692 assert!(disk.driver_exited);
15693 assert_eq!(disk.driver_pid, None);
15694 assert_eq!(disk.status, RunStatus::Blocked);
15695 }
15696
15697 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
15698 crate::run::MergeRepair {
15699 question: "repair-question".into(),
15700 head: "old-head".into(),
15701 pr: "https://github.com/o/r/pull/42".into(),
15702 reviews: vec![review_round(false, 1, 1, 1, false, true)],
15703 gate: vec![outcome(Some(0), false)],
15704 gate_head: Some("old-head".into()),
15705 gate_fixes: Vec::new(),
15706 pre_gate: Vec::new(),
15707 phase,
15708 failure: None,
15709 }
15710 }
15711
15712 #[tokio::test]
15713 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
15714 crate::run::pin_test_home();
15715 let mut runner = runner_at(RunStatus::Landing);
15716 runner.state.merge_repairs.push(merge_repair_request(
15717 crate::run::MergeRepairPhase::Requested,
15718 ));
15719 runner.state.save().unwrap();
15720 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
15721 assert!(
15722 runner
15723 .execute()
15724 .await
15725 .unwrap_err()
15726 .to_string()
15727 .contains("already running")
15728 );
15729 assert_eq!(
15730 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
15731 crate::run::MergeRepairPhase::Requested
15732 );
15733 drop(claim);
15734 let mut disk = runner.state.clone();
15735 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
15736 disk.save().unwrap();
15737 assert!(
15738 runner
15739 .execute()
15740 .await
15741 .unwrap_err()
15742 .to_string()
15743 .contains("decision changed")
15744 );
15745 assert_eq!(
15746 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
15747 crate::run::MergeRepairPhase::Complete
15748 );
15749 }
15750
15751 #[test]
15752 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
15753 crate::run::pin_test_home();
15754 let mut runner = runner_at(RunStatus::Landing);
15755 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
15756 runner.state.reviews = request.reviews.clone();
15757 runner.state.gate = request.gate.clone();
15758 runner.state.gate_ran = true;
15759 runner.state.land_approval = Some(crate::run::LandApproval {
15760 question: "old".into(),
15761 head: "old-head".into(),
15762 });
15763 runner.state.merge_repairs.push(request);
15764 runner.begin_merge_repair_review(0).unwrap();
15765 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
15766 assert!(!runner.state.gate_status().ok());
15767 assert!(runner.state.land_approval.is_none());
15768 let disk = RunState::load(&runner.state.id).unwrap();
15769 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
15770 assert!(disk.merge_repairs[0].gate[0].ok());
15771 assert_eq!(
15772 disk.merge_repairs[0].phase,
15773 crate::run::MergeRepairPhase::Reviewing
15774 );
15775 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
15776 assert!(disk.reviews.is_empty());
15777 }
15778
15779 #[tokio::test]
15780 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
15781 crate::run::pin_test_home();
15782 for phase in [
15783 crate::run::MergeRepairPhase::Fixing,
15784 crate::run::MergeRepairPhase::Requested,
15785 ] {
15786 let mut runner = runner_at(RunStatus::Landing);
15787 runner.state.config.graph.land_rounds = 0;
15788 runner.state.merge_repairs.push(merge_repair_request(phase));
15789 runner.state.save().unwrap();
15790 assert!(!runner.repair_merge_review().await.unwrap());
15791 assert_eq!(runner.state.status, RunStatus::Blocked);
15792 assert_eq!(
15793 runner.state.merge_repairs[0].phase,
15794 crate::run::MergeRepairPhase::Failed
15795 );
15796 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
15797 assert!(
15798 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
15799 "interrupted"
15800 } else {
15801 "budget exhausted"
15802 })
15803 );
15804 assert!(runner.state.merge.is_none());
15805 }
15806 }
15807
15808 #[test]
15812 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15813 crate::run::pin_test_home();
15814 let mut runner = runner_at(RunStatus::Implementing);
15815 let interrupt = Pause::new();
15816 runner.watch_interrupt(interrupt.clone());
15817
15818 interrupt.park_because("task a1b2 asked to run first");
15819
15820 assert!(runner.park_here().expect("park_here"));
15821 assert!(runner.state.parked);
15822 let last = runner.state.events.last().expect("a park event");
15823 assert_eq!(last.node, "park");
15824 assert!(
15825 last.message.contains("task a1b2 asked to run first"),
15826 "expected the interrupt reason in {:?}",
15827 last.message
15828 );
15829 }
15830
15831 #[test]
15834 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15835 crate::run::pin_test_home();
15836 let mut runner = runner_at(RunStatus::Implementing);
15837 runner.state.begin_pass();
15838 let shutdown = Pause::new();
15839 runner.on_pause(shutdown.clone());
15840 let upgrade = ParkCause::Upgrade {
15841 from: "0.121.0".to_owned(),
15842 to: Some("0.121.1".to_owned()),
15843 };
15844 shutdown.park_for(upgrade.clone());
15845 shutdown.park_for(ParkCause::Operator);
15846
15847 assert!(runner.park_here().expect("park_here"));
15848 assert_eq!(
15849 runner.state.passes[0].end,
15850 Some(crate::run::PassEnd::Parked { cause: upgrade })
15851 );
15852 let last = runner.state.events.last().expect("a park event");
15853 assert!(last.message.contains("0.121.0"), "{}", last.message);
15854 }
15855
15856 #[tokio::test]
15857 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15858 crate::run::pin_test_home();
15859 let mut runner = runner_at(RunStatus::Reviewing);
15860 runner.state.save().unwrap();
15861 let mut execution = runner.claim_driver().await.expect("first execution");
15862 assert_eq!(execution.runner.state.passes.len(), 1);
15863 execution
15864 .driver
15865 .finish(&mut execution.runner.state)
15866 .await
15867 .unwrap();
15868 }
15869
15870 #[tokio::test]
15871 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15872 crate::run::pin_test_home();
15873 let mut runner = runner_at(RunStatus::Reviewing);
15874 runner.state.config.agents = vec![conductor()];
15875 runner.state.config.graph.timeout_resource_wait = 0;
15876 runner.state.save().unwrap();
15877 let id = runner.state.id.clone();
15878 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15879 assert!(runner.claim_driver().await.is_err());
15880 drop(owner);
15881 drop(runner);
15882 let mut resumed = Runner::resume(&id).unwrap();
15883 assert_eq!(resumed.state.driver_claimed, Some(false));
15884 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15885 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15886 let mut execution = resumed
15887 .claim_driver()
15888 .await
15889 .expect("retry initial claim after reload");
15890 assert_eq!(execution.runner.state.driver_claimed, Some(true));
15891 assert_eq!(execution.runner.state.passes.len(), 1);
15892 execution
15893 .driver
15894 .finish(&mut execution.runner.state)
15895 .await
15896 .unwrap();
15897 }
15898
15899 #[tokio::test]
15900 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15901 crate::run::pin_test_home();
15902 let mut runner = runner_at(RunStatus::Reviewing);
15903 runner.state.config.agents = vec![conductor()];
15904 runner.state.save().unwrap();
15905 let id = runner.state.id.clone();
15906 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15907 let mut resumed = Runner::resume(&id).unwrap();
15908 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15909 assert!(resumed.claim_driver().await.is_err());
15910 assert!(RunState::load(&id).unwrap().passes.is_empty());
15911 drop(lease);
15912 let mut execution = resumed.claim_driver().await.unwrap();
15913 execution
15914 .driver
15915 .finish(&mut execution.runner.state)
15916 .await
15917 .unwrap();
15918 }
15919
15920 #[tokio::test]
15921 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15922 crate::run::pin_test_home();
15923 let mut runner = runner_at(RunStatus::Reviewing);
15924 let state = &mut runner.state;
15925 state.config.agents = vec![conductor()];
15926 state.driver_claimed = None;
15927 state.driver_pid = Some(std::process::id());
15928 state.driver_started_at = None;
15929 state.begin_pass();
15930 state.save().unwrap();
15931 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15932 std::fs::write(
15933 state.dir().join("driver.json"),
15934 serde_json::to_vec(&serde_json::json!({
15935 "pid": state.driver_pid, "started_at": null,
15936 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15937 }))
15938 .unwrap(),
15939 )
15940 .unwrap();
15941 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15942 drop(lease);
15943 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15944 let mut resumed = Runner::resume(&state.id).unwrap();
15945 assert!(!resumed.state.driver_exited);
15946 let mut execution = resumed
15947 .claim_driver()
15948 .await
15949 .expect("lease proves previous execution ended");
15950 assert_eq!(execution.runner.state.passes.len(), 2);
15951 execution
15952 .driver
15953 .finish(&mut execution.runner.state)
15954 .await
15955 .unwrap();
15956 }
15957
15958 #[tokio::test]
15959 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15960 crate::run::pin_test_home();
15961 let mut runner = runner_at(RunStatus::Reviewing);
15962 let mut json = serde_json::to_value(&runner.state).unwrap();
15963 json.as_object_mut().unwrap().remove("driver_claimed");
15964 runner.state = serde_json::from_value(json).unwrap();
15965 assert_eq!(runner.state.driver_claimed, None);
15966 runner.state.save().unwrap();
15967 let mut execution = runner
15968 .claim_driver()
15969 .await
15970 .expect("the execution lease, not a missing marker, excludes a second driver");
15971 assert_eq!(execution.runner.state.passes.len(), 1);
15972 execution
15973 .driver
15974 .finish(&mut execution.runner.state)
15975 .await
15976 .unwrap();
15977 }
15978
15979 #[tokio::test]
15980 async fn driver_admission_refuses_a_proven_live_driver() {
15981 crate::run::pin_test_home();
15982 let mut runner = runner_at(RunStatus::Reviewing);
15983 let pid = std::process::id();
15984 runner.state.driver_pid = Some(pid);
15985 runner.state.driver_started_at = crate::proc::process_started_at(pid);
15986 assert!(runner.state.driver_started_at.is_some());
15987 runner.state.save().unwrap();
15988 let error = runner
15989 .claim_driver()
15990 .await
15991 .err()
15992 .expect("a live driver must refuse");
15993 assert!(error.to_string().contains("live driver"), "{error:#}");
15994 let saved = RunState::load(&runner.state.id).unwrap();
15995 assert_eq!(saved.driver_pid, Some(pid));
15996 assert_eq!(saved.status, RunStatus::Reviewing);
15997 assert!(saved.passes.is_empty());
15998 }
15999
16000 #[test]
16001 fn parking_resource_halt_persists_the_pass_once() {
16002 crate::run::pin_test_home();
16003 let mut runner = runner_at(RunStatus::Reviewing);
16004 runner.state.begin_pass();
16005 runner.state.resource_block = Some(crate::run::ResourceBlock {
16006 node: "fix".into(),
16007 kind: "shared build cache".into(),
16008 detail: "cancelled".into(),
16009 at: Timestamp::now(),
16010 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16011 });
16012 let cause = ParkCause::Upgrade {
16013 from: "old".into(),
16014 to: Some("new".into()),
16015 };
16016 runner.pause.park_for(cause.clone());
16017 runner.stop_for_resource().unwrap();
16020 assert!(runner.resource_halt().unwrap());
16021 assert!(runner.park_here().unwrap());
16022 let saved = RunState::load(&runner.state.id).unwrap();
16023 assert!(saved.parked);
16024 assert_eq!(saved.status, RunStatus::Reviewing);
16025 assert!(saved.passes[0].ended_at.is_some());
16026 assert_eq!(
16027 saved.passes[0].end,
16028 Some(crate::run::PassEnd::Parked {
16029 cause: cause.clone()
16030 })
16031 );
16032 assert_eq!(
16033 saved.resource_block.unwrap().cause,
16034 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16035 );
16036 assert_eq!(
16037 saved
16038 .events
16039 .iter()
16040 .filter(|event| event.node == "park")
16041 .count(),
16042 1
16043 );
16044 }
16045
16046 #[test]
16054 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16055 crate::run::pin_test_home();
16056 let mut runner = runner_at(RunStatus::Implementing);
16057 let shutdown = Pause::new();
16058 runner.on_pause(shutdown.clone());
16059 let interrupt = Pause::new();
16060 runner.watch_interrupt(interrupt.clone());
16061
16062 assert!(!runner.park_here().expect("park_here"));
16064 assert!(!runner.state.parked);
16065
16066 interrupt.park_because("test");
16068 assert!(!shutdown.parked());
16069 assert!(runner.park_here().expect("park_here"));
16070 }
16071
16072 #[tokio::test]
16086 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16087 crate::run::pin_test_home();
16088 let mut runner = runner_at(RunStatus::Implementing);
16089 let interrupt = Pause::new();
16090 runner.watch_interrupt(interrupt.clone());
16091
16092 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16093 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16094
16095 let node = async move {
16099 started_tx.send(()).expect("send started");
16100 finish_rx.await.expect("recv finish");
16101 "node finished"
16102 };
16103
16104 let interrupter = async move {
16105 started_rx.await.expect("recv started");
16106 interrupt.park_because("higher-priority task waiting");
16108 tokio::task::yield_now().await;
16112 finish_tx.send(()).expect("send finish");
16113 };
16114
16115 let (node_result, ()) = tokio::join!(node, interrupter);
16116 assert_eq!(
16117 node_result, "node finished",
16118 "the in-flight call ran to completion"
16119 );
16120
16121 assert!(runner.park_here().expect("park_here"));
16124 assert!(runner.state.parked);
16125 }
16126
16127 #[test]
16133 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16134 crate::run::pin_test_home();
16135 let mut runner = runner_at(RunStatus::Judging);
16136 runner.state.config.agents = vec![conductor()];
16140 runner.state.candidates = vec![Candidate {
16141 index: 0,
16142 label: 'A',
16143 agent: "alpha".to_owned(),
16144 branch: "magi/x/A".to_owned(),
16145 worktree: PathBuf::from("/nonexistent/worktree"),
16146 summary: "did the thing".to_owned(),
16147 stat: "1 file changed".to_owned(),
16148 files: 1,
16149 commits: 1,
16150 empty: false,
16151 failed: None,
16152 verified_noop: None,
16153 duration_ms: 1234,
16154 folded: false,
16155 }];
16156 let run_id = runner.state.id.clone();
16157
16158 let interrupt = Pause::new();
16159 runner.watch_interrupt(interrupt.clone());
16160 interrupt.park_because("task c3d4 asked to run first");
16161 assert!(runner.park_here().expect("park_here"));
16162
16163 let resumed = Runner::resume(&run_id).expect("resume");
16164 assert_eq!(resumed.state.candidates.len(), 1);
16165 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16166 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16167 assert_eq!(resumed.state.status, runner.state.status);
16168 assert!(
16169 resumed.state.parked,
16170 "still parked until `execute` actually walks the graph again"
16171 );
16172 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16173 }
16174
16175 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16177 let mut q = ask::Question::new(
16178 run.to_owned(),
16179 "implement".to_owned(),
16180 "impl-A".to_owned(),
16181 "Which storage backend should the cache use?".to_owned(),
16182 String::new(),
16183 vec!["SQLite".to_owned(), "Redis".to_owned()],
16184 );
16185 store.put(&mut q).unwrap();
16186 q
16187 }
16188
16189 #[test]
16190 fn a_failed_runs_open_question_is_abandoned() {
16191 ask_test_home();
16192 let store = ask::Questions::open();
16193 let mut runner = runner_at(RunStatus::Failed);
16194 let run = runner.state.id.clone();
16195 let q = ask_open_question(&store, &run);
16196
16197 runner.settle_questions();
16198
16199 let back = store.get(&q.id).unwrap();
16200 assert!(
16201 !back.status.open(),
16202 "the seat that asked died with the run; nobody is left to read an answer"
16203 );
16204 assert!(
16205 back.detail.contains(&run) && back.detail.contains("failed"),
16206 "the reason names what the run became, not just that it is gone: {}",
16207 back.detail
16208 );
16209 }
16210
16211 #[test]
16212 fn a_merged_runs_open_question_is_abandoned_too() {
16213 ask_test_home();
16214 let store = ask::Questions::open();
16215 for status in [RunStatus::Merged, RunStatus::Ready] {
16218 let mut runner = runner_at(status);
16219 let run = runner.state.id.clone();
16220 let q = ask_open_question(&store, &run);
16221
16222 runner.settle_questions();
16223
16224 let back = store.get(&q.id).unwrap();
16225 assert!(
16226 !back.status.open(),
16227 "{status:?} run's question must not outlive the run"
16228 );
16229 }
16230 }
16231
16232 #[test]
16233 fn a_still_resumable_runs_open_question_is_left_alone() {
16234 ask_test_home();
16235 let store = ask::Questions::open();
16236 for status in [RunStatus::Blocked, RunStatus::Stalled] {
16242 let mut runner = runner_at(status);
16243 let run = runner.state.id.clone();
16244 let q = ask_open_question(&store, &run);
16245
16246 runner.settle_questions();
16247
16248 let back = store.get(&q.id).unwrap();
16249 assert!(
16250 back.status.open(),
16251 "{status:?} is still alive; the question must still be waiting"
16252 );
16253 }
16254 }
16255
16256 #[test]
16257 fn settle_questions_never_touches_an_already_answered_question() {
16258 ask_test_home();
16259 let store = ask::Questions::open();
16260 let mut runner = runner_at(RunStatus::Failed);
16261 let run = runner.state.id.clone();
16262 let mut q = ask_open_question(&store, &run);
16263 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16264 .unwrap();
16265 store.put(&mut q).unwrap();
16266
16267 runner.settle_questions();
16272 runner.settle_questions();
16273
16274 let back = store.get(&q.id).unwrap();
16275 assert_eq!(
16276 back.status,
16277 ask::QuestionStatus::Answered,
16278 "a real answer is a decision on record, never overwritten by a sweep"
16279 );
16280 }
16281
16282 #[tokio::test]
16293 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16294 crate::run::pin_test_home();
16295 let tmp = tempfile::tempdir().expect("tempdir");
16296 let repo = tmp.path().join("repo");
16297 std::fs::create_dir_all(&repo).unwrap();
16298 init_repo(&repo);
16299
16300 let mut config = Config::default();
16301 config.graph.worktree_root = Some(tmp.path().join("wt"));
16302
16303 let mut state = RunState::new(
16304 repo.clone(),
16305 "main".to_owned(),
16306 "deadbeef".to_owned(),
16307 "task".to_owned(),
16308 config,
16309 );
16310 let root = state.worktree_root();
16311 let wt_a = root.join("cand-A");
16312 let wt_b = root.join("cand-B");
16313 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16314 .await
16315 .expect("worktree A");
16316 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16317 .await
16318 .expect("worktree B");
16319
16320 state.candidates = vec![
16321 Candidate {
16322 index: 0,
16323 label: 'A',
16324 agent: "alpha".to_owned(),
16325 branch: "magi/x/A".to_owned(),
16326 worktree: wt_a.clone(),
16327 summary: String::new(),
16328 stat: String::new(),
16329 files: 0,
16330 commits: 0,
16331 empty: false,
16332 failed: None,
16333 verified_noop: None,
16334 duration_ms: 0,
16335 folded: false,
16336 },
16337 Candidate {
16338 index: 1,
16339 label: 'B',
16340 agent: "beta".to_owned(),
16341 branch: "magi/x/B".to_owned(),
16342 worktree: wt_b.clone(),
16343 summary: String::new(),
16344 stat: String::new(),
16345 files: 0,
16346 commits: 0,
16347 empty: false,
16348 failed: None,
16349 verified_noop: None,
16350 duration_ms: 0,
16351 folded: false,
16352 },
16353 ];
16354 state.tally = Some(Tally {
16355 first_choice: BTreeMap::from([('A', 1)]),
16356 borda: BTreeMap::new(),
16357 winner: 'A',
16358 rankings: 1,
16359 unanimous_initial: true,
16360 deliberated: false,
16361 changed_votes: 0,
16362 unanimous_final: true,
16363 tie_break: None,
16364 judges: 1,
16365 present: 1,
16366 quorum: 1,
16367 met_quorum: true,
16368 uncontested: None,
16369 });
16370 state.status = RunStatus::Ready;
16371
16372 fold_run(&mut state, false, &crate::run::home())
16373 .await
16374 .expect("fold_run");
16375
16376 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
16377 assert!(
16378 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16379 "the unmerged winner's branch survives"
16380 );
16381 assert!(
16382 !state.candidates[0].folded,
16383 "the winner is not marked folded"
16384 );
16385
16386 assert!(!wt_b.exists(), "the loser's worktree is removed");
16387 assert!(
16388 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
16389 "the loser's branch is removed"
16390 );
16391 assert!(state.candidates[1].folded, "the loser is marked folded");
16392 }
16393
16394 #[tokio::test]
16395 #[cfg(unix)]
16396 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
16397 use std::os::fd::AsRawFd as _;
16398 use std::time::Duration;
16399
16400 let tmp = tempfile::tempdir().unwrap();
16401 let repo = tmp.path().join("repo");
16402 std::fs::create_dir_all(&repo).unwrap();
16403 init_repo(&repo);
16404 let common = repo.join(".git");
16405 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
16406 let home = tmp.path().join("home");
16407 let mut config = Config::default();
16408 config.coordination = crate::config::test_coordination();
16409 config.graph.worktree_root = Some(tmp.path().join("wt"));
16410 config.graph.timeout_resource_wait = 1;
16411 let mut state = RunState::new(
16412 repo.clone(),
16413 "main".into(),
16414 "deadbeef".into(),
16415 "task".into(),
16416 config,
16417 );
16418 let root = state.worktree_root();
16419 let candidate = root.join("cand-A");
16420 let reviewer = root.join("review-1");
16421 let branch = state.branch_for('A');
16422 git::worktree_add_branch(&repo, &candidate, &branch, "main")
16423 .await
16424 .unwrap();
16425 git::worktree_add_detached(&repo, &reviewer, "main")
16426 .await
16427 .unwrap();
16428 git::acquire_worktree_config(&repo).await.unwrap();
16429 state.enabled_worktree_config = true;
16430 state.status = RunStatus::Ready;
16431 state.candidates.push(Candidate {
16432 index: 0,
16433 label: 'A',
16434 agent: "implement".into(),
16435 branch: branch.clone(),
16436 worktree: candidate.clone(),
16437 summary: String::new(),
16438 stat: String::new(),
16439 files: 0,
16440 commits: 0,
16441 empty: false,
16442 failed: None,
16443 verified_noop: None,
16444 duration_ms: 0,
16445 folded: false,
16446 });
16447 state.save_under(&home).unwrap();
16448 let config_before = std::fs::read(common.join("config")).unwrap();
16449 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
16450 .await
16451 .unwrap();
16452 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
16455 assert_eq!(
16456 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
16457 0
16458 );
16459 let mut fold = Box::pin(fold_run(&mut state, true, &home));
16460 assert!(
16461 tokio::time::timeout(Duration::from_millis(200), &mut fold)
16462 .await
16463 .is_err()
16464 );
16465 assert!(candidate.exists());
16466 assert!(reviewer.exists());
16467 assert!(git::branch_exists(&repo, &branch).await.unwrap());
16468 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
16469 assert_eq!(
16470 git::git(&repo, &["worktree", "list", "--porcelain"])
16471 .await
16472 .unwrap(),
16473 worktrees_before
16474 );
16475 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
16476 .await
16477 .unwrap()
16478 .unwrap_err();
16479 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
16480 drop(fold);
16481 assert!(candidate.exists() && reviewer.exists());
16482 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
16483 assert_eq!(
16484 state.resource_block.as_ref().unwrap().kind,
16485 "repository coordination"
16486 );
16487 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
16488 drop(external);
16489 let mut fold = Box::pin(fold_run(&mut state, true, &home));
16490 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
16491 .await
16492 .unwrap()
16493 .unwrap();
16494 drop(fold);
16495 assert!(removed.contains(&branch));
16496 assert!(!root.exists());
16497 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
16498 assert!(state.candidates[0].folded);
16499 assert!(!state.enabled_worktree_config);
16500 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
16501 let enabled = git::git_raw(
16502 &repo,
16503 &["config", "--local", "--get", "extensions.worktreeConfig"],
16504 )
16505 .await
16506 .unwrap();
16507 assert!(!enabled.ok(), "the last config reference was released");
16508 }
16509
16510 #[tokio::test]
16513 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
16514 let tmp = tempfile::tempdir().expect("tempdir");
16515 let repo = tmp.path().join("repo");
16516 std::fs::create_dir_all(&repo).unwrap();
16517 init_repo(&repo);
16518 let home = tmp.path().join("home");
16519
16520 let mut config = Config::default();
16521 config.graph.worktree_root = Some(tmp.path().join("wt"));
16522 let mut state = RunState::new(
16523 repo.clone(),
16524 "main".to_owned(),
16525 "deadbeef".to_owned(),
16526 "task".to_owned(),
16527 config,
16528 );
16529 git::git(&repo, &["branch", "magi/x/A", "main"])
16531 .await
16532 .expect("branch");
16533 state.candidates = vec![Candidate {
16534 index: 0,
16535 label: 'A',
16536 agent: "alpha".to_owned(),
16537 branch: "magi/x/A".to_owned(),
16538 worktree: state.worktree_root().join("cand-A"),
16539 summary: String::new(),
16540 stat: String::new(),
16541 files: 0,
16542 commits: 0,
16543 empty: false,
16544 failed: None,
16545 verified_noop: None,
16546 duration_ms: 0,
16547 folded: true,
16548 }];
16549 state.released_to = Some("20260901-000000-new1".to_owned());
16550 state.released_branches = vec!["magi/x/A".to_owned()];
16551
16552 fold_run(&mut state, true, &home).await.expect("fold_run");
16553 state.status = RunStatus::Merged;
16554 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
16555 state.merge = Some(crate::run::MergeOutcome {
16556 mode: MergeMode::Pr,
16557 ok: true,
16558 detail: "confirmed merged".into(),
16559 empty: false,
16560 });
16561 fold_run(&mut state, true, &home)
16562 .await
16563 .expect("merged fold");
16564
16565 assert!(
16566 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16567 "the handed-over branch survives a fold"
16568 );
16569 }
16570
16571 #[tokio::test]
16575 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
16576 let tmp = tempfile::tempdir().expect("tempdir");
16577 let repo = tmp.path().join("repo");
16578 std::fs::create_dir_all(&repo).unwrap();
16579 init_repo(&repo);
16580 let run = |args: &[&str]| {
16581 let out = std::process::Command::new("git")
16582 .quiet()
16583 .args(args)
16584 .current_dir(&repo)
16585 .output()
16586 .expect("spawn git");
16587 assert!(out.status.success(), "git {args:?}");
16588 };
16589 run(&["branch", "magi/x/A"]);
16590 run(&["checkout", "-q", "-b", "magi/x/B"]);
16591 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
16592 run(&["add", "-A"]);
16593 run(&["commit", "-q", "-m", "work"]);
16594 run(&["checkout", "-q", "main"]);
16595
16596 let probe = RunState::new(
16600 repo.clone(),
16601 "main".to_owned(),
16602 "deadbeef".to_owned(),
16603 "task".to_owned(),
16604 Config::default(),
16605 );
16606 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
16607 let bare = tmp.path().join("origin.git");
16608 let out = std::process::Command::new("git")
16609 .quiet()
16610 .args(["init", "-q", "--bare"])
16611 .arg(&bare)
16612 .output()
16613 .expect("spawn git");
16614 assert!(out.status.success(), "git init --bare");
16615 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
16616 run(&["push", "-q", "origin", "main"]);
16617
16618 let mut state = RunState::new(
16619 repo.clone(),
16620 "main".to_owned(),
16621 "deadbeef".to_owned(),
16622 "task".to_owned(),
16623 Config::default(),
16624 );
16625 state.seeds = vec![refs::Seed {
16626 token: "magi/27b2/A".to_owned(),
16627 kind: refs::SeedKind::Unresolved,
16628 sha: String::new(),
16629 branch: true,
16630 detail: "no branch or commit named magi/27b2/A".to_owned(),
16631 }];
16632
16633 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
16634 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
16635 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
16636 let detail = empty_candidate_detail(&state, "main");
16637 assert!(detail.starts_with("empty candidate"), "{detail}");
16638 assert!(!detail.contains("magi/27b2/A"), "{detail}");
16639 assert!(detail.contains("--start-from"), "{detail}");
16640 }
16641
16642 #[tokio::test]
16651 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
16652 let tmp = tempfile::tempdir().expect("tempdir");
16653 let repo = tmp.path().join("repo");
16654 std::fs::create_dir_all(&repo).unwrap();
16655 init_repo(&repo);
16656
16657 let mut config = Config::default();
16658 config.merge.mode = MergeMode::Local;
16659
16660 let mut state = RunState::new(
16661 repo.clone(),
16662 "main".to_owned(),
16663 "deadbeef".to_owned(),
16664 "task".to_owned(),
16665 config,
16666 );
16667 state.candidates = vec![Candidate {
16668 index: 0,
16669 label: 'A',
16670 agent: "alpha".to_owned(),
16671 branch: "does-not-exist".to_owned(),
16672 worktree: repo.clone(),
16673 summary: String::new(),
16674 stat: String::new(),
16675 files: 0,
16676 commits: 0,
16677 empty: false,
16678 failed: None,
16679 verified_noop: None,
16680 duration_ms: 0,
16681 folded: false,
16682 }];
16683 state.tally = Some(Tally {
16684 first_choice: BTreeMap::from([('A', 1)]),
16685 borda: BTreeMap::new(),
16686 winner: 'A',
16687 rankings: 1,
16688 unanimous_initial: true,
16689 deliberated: false,
16690 changed_votes: 0,
16691 unanimous_final: true,
16692 tie_break: None,
16693 judges: 0,
16694 present: 0,
16695 quorum: 0,
16696 met_quorum: true,
16697 uncontested: Some("only candidate A produced a change".to_owned()),
16698 });
16699 state.reviews = vec![ReviewRound {
16700 round: 1,
16701 head: "deadbeef".to_owned(),
16702 verified_head: None,
16703 verified_at: None,
16704 reviews: Vec::new(),
16705 e2e: Vec::new(),
16706 fix: None,
16707 blocking: 0,
16708 answered: 0,
16709 expected: 0,
16710 clean: true,
16711 verify_retried: false,
16712 e2e_deferred: false,
16713 e2e_defer_reason: None,
16714 progressed: false,
16715 vote_split: false,
16716 reconsideration: Vec::new(),
16717 verdict: None,
16718 }];
16719 state.gate = vec![CommandOutcome {
16720 command: "test".to_owned(),
16721 code: Some(0),
16722 output_tail: String::new(),
16723 duration_ms: 0,
16724 resource_blocked: false,
16725 }];
16726 state.gate_ran = true;
16727 state.status = RunStatus::Ready;
16732 state.merge = Some(MergeOutcome {
16733 mode: MergeMode::Local,
16734 ok: false,
16735 detail: "already concluded".to_owned(),
16736 empty: false,
16737 });
16738
16739 let mut runner = Runner {
16740 state,
16741 roles: ResolvedRoles {
16742 implementers: Vec::new(),
16743 judges: Vec::new(),
16744 reviewers: Vec::new(),
16745 fixer: None,
16746 conductor: conductor(),
16747 implementer_roster: Vec::new(),
16748 judge_roster: Vec::new(),
16749 reviewer_roster: Vec::new(),
16750 },
16751 sem: Arc::new(Semaphore::new(1)),
16752 pause: Pause::new(),
16753 interrupt: Pause::new(),
16754 };
16755
16756 runner.merge().await.expect("merge");
16757
16758 assert_eq!(
16759 runner.state.status,
16760 RunStatus::Ready,
16761 "a concluded run's status must not change on reentry"
16762 );
16763 assert_eq!(
16764 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16765 Some("already concluded"),
16766 "merge must not run again once the node already recorded an outcome"
16767 );
16768 }
16769
16770 #[tokio::test]
16779 async fn merge_refuses_a_gate_that_has_not_actually_run() {
16780 let tmp = tempfile::tempdir().expect("tempdir");
16781 let repo = tmp.path().join("repo");
16782 std::fs::create_dir_all(&repo).unwrap();
16783 init_repo(&repo);
16784
16785 let mut config = Config::default();
16786 config.merge.mode = MergeMode::Local;
16787
16788 let mut state = RunState::new(
16789 repo.clone(),
16790 "main".to_owned(),
16791 "deadbeef".to_owned(),
16792 "task".to_owned(),
16793 config,
16794 );
16795 state.candidates = vec![Candidate {
16796 index: 0,
16797 label: 'A',
16798 agent: "alpha".to_owned(),
16799 branch: "does-not-exist".to_owned(),
16800 worktree: repo.clone(),
16801 summary: String::new(),
16802 stat: String::new(),
16803 files: 0,
16804 commits: 0,
16805 empty: false,
16806 failed: None,
16807 verified_noop: None,
16808 duration_ms: 0,
16809 folded: false,
16810 }];
16811 state.tally = Some(Tally {
16812 first_choice: BTreeMap::from([('A', 1)]),
16813 borda: BTreeMap::new(),
16814 winner: 'A',
16815 rankings: 1,
16816 unanimous_initial: true,
16817 deliberated: false,
16818 changed_votes: 0,
16819 unanimous_final: true,
16820 tie_break: None,
16821 judges: 0,
16822 present: 0,
16823 quorum: 0,
16824 met_quorum: true,
16825 uncontested: Some("only candidate A produced a change".to_owned()),
16826 });
16827 state.reviews = vec![ReviewRound {
16828 round: 1,
16829 head: "deadbeef".to_owned(),
16830 verified_head: None,
16831 verified_at: None,
16832 reviews: Vec::new(),
16833 e2e: Vec::new(),
16834 fix: None,
16835 blocking: 0,
16836 answered: 0,
16837 expected: 0,
16838 clean: true,
16839 verify_retried: false,
16840 e2e_deferred: false,
16841 e2e_defer_reason: None,
16842 progressed: false,
16843 vote_split: false,
16844 reconsideration: Vec::new(),
16845 verdict: None,
16846 }];
16847 state.gate = Vec::new();
16849 state.gate_ran = false;
16850 state.status = RunStatus::Gating;
16851
16852 let mut runner = Runner {
16853 state,
16854 roles: ResolvedRoles {
16855 implementers: Vec::new(),
16856 judges: Vec::new(),
16857 reviewers: Vec::new(),
16858 fixer: None,
16859 conductor: conductor(),
16860 implementer_roster: Vec::new(),
16861 judge_roster: Vec::new(),
16862 reviewer_roster: Vec::new(),
16863 },
16864 sem: Arc::new(Semaphore::new(1)),
16865 pause: Pause::new(),
16866 interrupt: Pause::new(),
16867 };
16868
16869 runner.merge().await.expect("merge");
16870
16871 assert!(
16872 runner.state.merge.is_none(),
16873 "an empty gate must never be read as a passing one: {:?}",
16874 runner.state.merge
16875 );
16876 }
16877
16878 #[tokio::test]
16885 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16886 let tmp = tempfile::tempdir().expect("tempdir");
16887 let repo = tmp.path().join("repo");
16888 std::fs::create_dir_all(&repo).unwrap();
16889 init_repo(&repo);
16890
16891 let config = Config::default();
16893
16894 let mut state = RunState::new(
16895 repo.clone(),
16896 "main".to_owned(),
16897 "deadbeef".to_owned(),
16898 "task".to_owned(),
16899 config,
16900 );
16901 state.candidates = vec![Candidate {
16902 index: 0,
16903 label: 'A',
16904 agent: "alpha".to_owned(),
16905 branch: "does-not-exist".to_owned(),
16906 worktree: repo.clone(),
16907 summary: String::new(),
16908 stat: String::new(),
16909 files: 0,
16910 commits: 0,
16911 empty: false,
16912 failed: None,
16913 verified_noop: None,
16914 duration_ms: 0,
16915 folded: false,
16916 }];
16917 state.tally = Some(Tally {
16918 first_choice: BTreeMap::from([('A', 1)]),
16919 borda: BTreeMap::new(),
16920 winner: 'A',
16921 rankings: 1,
16922 unanimous_initial: true,
16923 deliberated: false,
16924 changed_votes: 0,
16925 unanimous_final: true,
16926 tie_break: None,
16927 judges: 0,
16928 present: 0,
16929 quorum: 0,
16930 met_quorum: true,
16931 uncontested: Some("only candidate A produced a change".to_owned()),
16932 });
16933 state.reviews = vec![ReviewRound {
16934 round: 1,
16935 head: "deadbeef".to_owned(),
16936 verified_head: None,
16937 verified_at: None,
16938 reviews: Vec::new(),
16939 e2e: Vec::new(),
16940 fix: None,
16941 blocking: 0,
16942 answered: 0,
16943 expected: 0,
16944 clean: true,
16945 verify_retried: false,
16946 e2e_deferred: false,
16947 e2e_defer_reason: None,
16948 progressed: false,
16949 vote_split: false,
16950 reconsideration: Vec::new(),
16951 verdict: None,
16952 }];
16953
16954 let mut runner = Runner {
16955 state,
16956 roles: ResolvedRoles {
16957 implementers: Vec::new(),
16958 judges: Vec::new(),
16959 reviewers: Vec::new(),
16960 fixer: None,
16961 conductor: conductor(),
16962 implementer_roster: Vec::new(),
16963 judge_roster: Vec::new(),
16964 reviewer_roster: Vec::new(),
16965 },
16966 sem: Arc::new(Semaphore::new(1)),
16967 pause: Pause::new(),
16968 interrupt: Pause::new(),
16969 };
16970
16971 runner.gate().await.expect("gate");
16972 assert!(
16973 runner.state.gate_ran,
16974 "zero configured commands is still a real attempt, not an unrun gate"
16975 );
16976 assert!(runner.state.gate.is_empty());
16977 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16978 assert_ne!(
16979 runner.state.status,
16980 RunStatus::Blocked,
16981 "a gate with nothing to check must not read as failed"
16982 );
16983
16984 runner.merge().await.expect("merge");
16985 assert_eq!(
16986 runner.state.status,
16987 RunStatus::Ready,
16988 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16989 );
16990 }
16991
16992 #[tokio::test]
17003 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17004 crate::run::pin_test_home();
17005 let home = crate::run::home();
17006
17007 let tmp = tempfile::tempdir().expect("tempdir");
17008 let repo = tmp.path().join("repo");
17009 std::fs::create_dir_all(&repo).unwrap();
17010 init_repo(&repo);
17011 let cache_dir = tmp.path().join("target");
17014
17015 let mut config = Config::default();
17016 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17017 config.graph.timeout_verify = Some(2);
17020 config.graph.timeout_resource_wait = 2;
17021
17022 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17023 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17024 .expect("no io error acquiring directly")
17025 {
17026 crate::cache::AcquireOutcome::Acquired(g) => g,
17027 crate::cache::AcquireOutcome::Busy(b) => {
17028 panic!("expected the direct acquire to win the lease first: {b:?}")
17029 }
17030 };
17031
17032 let mut state = RunState::new(
17033 repo.clone(),
17034 "main".to_owned(),
17035 "deadbeef".to_owned(),
17036 "task".to_owned(),
17037 config,
17038 );
17039 state.candidates = vec![Candidate {
17040 index: 0,
17041 label: 'A',
17042 agent: "alpha".to_owned(),
17043 branch: "does-not-exist".to_owned(),
17044 worktree: repo.clone(),
17045 summary: String::new(),
17046 stat: String::new(),
17047 files: 0,
17048 commits: 0,
17049 empty: false,
17050 failed: None,
17051 verified_noop: None,
17052 duration_ms: 0,
17053 folded: false,
17054 }];
17055 state.tally = Some(Tally {
17056 first_choice: BTreeMap::from([('A', 1)]),
17057 borda: BTreeMap::new(),
17058 winner: 'A',
17059 rankings: 1,
17060 unanimous_initial: true,
17061 deliberated: false,
17062 changed_votes: 0,
17063 unanimous_final: true,
17064 tie_break: None,
17065 judges: 0,
17066 present: 0,
17067 quorum: 0,
17068 met_quorum: true,
17069 uncontested: Some("only candidate A produced a change".to_owned()),
17070 });
17071 state.reviews = vec![ReviewRound {
17072 round: 1,
17073 head: "deadbeef".to_owned(),
17074 verified_head: None,
17075 verified_at: None,
17076 reviews: Vec::new(),
17077 e2e: Vec::new(),
17078 fix: None,
17079 blocking: 0,
17080 answered: 0,
17081 expected: 0,
17082 clean: true,
17083 verify_retried: false,
17084 e2e_deferred: false,
17085 e2e_defer_reason: None,
17086 progressed: false,
17087 vote_split: false,
17088 reconsideration: Vec::new(),
17089 verdict: None,
17090 }];
17091
17092 let mut runner = Runner {
17093 state,
17094 roles: ResolvedRoles {
17095 implementers: Vec::new(),
17096 judges: Vec::new(),
17097 reviewers: Vec::new(),
17098 fixer: None,
17099 conductor: conductor(),
17100 implementer_roster: Vec::new(),
17101 judge_roster: Vec::new(),
17102 reviewer_roster: Vec::new(),
17103 },
17104 sem: Arc::new(Semaphore::new(1)),
17105 pause: Pause::new(),
17106 interrupt: Pause::new(),
17107 };
17108
17109 let started = std::time::Instant::now();
17110 runner.gate().await.expect("gate");
17111 assert!(
17112 started.elapsed() < Duration::from_secs(1),
17113 "a gate with nothing to run must never wait on a lease it never needed"
17114 );
17115 assert!(
17116 runner.state.gate_ran,
17117 "zero commands is still a real, immediate attempt"
17118 );
17119 assert!(runner.state.gate.is_empty());
17120 assert_ne!(
17121 runner.state.status,
17122 RunStatus::Blocked,
17123 "must not read as resource-blocked on a lease it never asked for"
17124 );
17125 }
17126
17127 #[cfg(unix)]
17130 #[tokio::test]
17131 async fn verification_is_standalone_inside_a_live_worker() {
17132 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17133 if std::env::var_os(CHILD).is_none() {
17134 let home = tempfile::tempdir().unwrap();
17135 let generation = crate::worker::pin_current(home.path()).unwrap();
17136 let output = std::process::Command::new(std::env::current_exe().unwrap())
17137 .args([
17138 "--exact",
17139 "graph::tests::verification_is_standalone_inside_a_live_worker",
17140 "--nocapture",
17141 ])
17142 .env(CHILD, "1")
17143 .env("MAGI_HOME", home.path())
17144 .env("MAGI_WORKER_HOME", home.path())
17145 .env("MAGI_WORKER_ID", "abcdef")
17146 .env("MAGI_WORKER_GENERATION", &generation.id)
17147 .env("MAGI_WORKER_EXE", &generation.executable)
17148 .env("MAGI_RUN", "verification-parent")
17149 .env("MAGI_NODE", "gate")
17150 .env("MAGI_SEAT", "worker-fixture")
17151 .env("MAGI_PROCESS_OWNER", "fixture-owner")
17152 .env("MAGI_NO_AUTOUPDATE", "0")
17153 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17154 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17155 .env(
17156 "VERIFY_EXPECT_TOOLCHAIN",
17157 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17158 )
17159 .env(
17160 "VERIFY_EXPECT_CACHE",
17161 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17162 )
17163 .output()
17164 .unwrap();
17165 assert!(
17166 output.status.success()
17167 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17168 "{}\n{}",
17169 String::from_utf8_lossy(&output.stdout),
17170 String::from_utf8_lossy(&output.stderr)
17171 );
17172 return;
17173 }
17174 let home = crate::run::home();
17175 let directory = home.join("workers/jobs/abcdef");
17176 std::fs::create_dir_all(&directory).unwrap();
17177 let job = crate::worker::Job {
17178 id: "abcdef".into(),
17179 sequence: 1,
17180 kind: "task".into(),
17181 key: "task:verification".into(),
17182 payload: serde_json::json!({}),
17183 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17184 version: Some(env!("CARGO_PKG_VERSION").into()),
17185 status: crate::worker::Status::Running,
17186 result: None,
17187 pid: Some(std::process::id()),
17188 created_at: 0,
17189 finished_at: None,
17190 };
17191 std::fs::write(
17192 directory.join("job.json"),
17193 serde_json::to_vec(&job).unwrap(),
17194 )
17195 .unwrap();
17196 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17197 let assert_worker = || {
17198 assert_eq!(
17199 crate::worker::current_job(&home).unwrap().unwrap().id,
17200 job.id
17201 );
17202 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17203 assert_eq!(
17204 crate::worker::helper_executable(&home).unwrap(),
17205 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17206 );
17207 assert!(
17208 crate::updater::Checker::new(&crate::config::Update {
17209 mode: crate::config::UpdateMode::Notify,
17210 interval: None,
17211 auto_restart: false,
17212 })
17213 .is_none(),
17214 "the driver remains pinned, even while tests are standalone"
17215 );
17216 };
17217 assert_worker();
17218 let repo = home.join("repo");
17219 std::fs::create_dir_all(&repo).unwrap();
17220 init_repo(&repo);
17221 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17224 test -z "${MAGI_WORKER_GENERATION+x}" &&
17225 test -z "${MAGI_WORKER_HOME+x}" &&
17226 test -z "${MAGI_WORKER_EXE+x}" &&
17227 test "$MAGI_HOME/repo" -ef . &&
17228 test "$MAGI_RUN" = verification-parent &&
17229 test "$MAGI_NODE" = gate &&
17230 test "$MAGI_SEAT" = worker-fixture &&
17231 test -n "$MAGI_PROCESS_OWNER" &&
17232 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17233 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17234 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17235 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17236 assert!(
17238 !std::process::Command::new("sh")
17239 .args(["-c", probe])
17240 .current_dir(&repo)
17241 .status()
17242 .unwrap()
17243 .success()
17244 );
17245 let unrelated_home = tempfile::tempdir().unwrap();
17246 assert!(
17247 crate::worker::current_job(unrelated_home.path()).is_err(),
17248 "an inherited worker id cannot be resolved in a unit fixture's home"
17249 );
17250 let mut runner = runner_at(RunStatus::Gating);
17251 runner.state.repo = repo.clone();
17252 runner.state.config.disk.min_free_bytes = 0;
17253 runner.state.config.coordination = crate::config::test_coordination();
17254 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17255 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17256 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17257 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17258 let winner = Candidate {
17259 index: 0,
17260 label: 'A',
17261 agent: "fixture".into(),
17262 branch: "main".into(),
17263 worktree: repo.clone(),
17264 summary: String::new(),
17265 stat: String::new(),
17266 files: 0,
17267 commits: 0,
17268 empty: false,
17269 failed: None,
17270 verified_noop: None,
17271 duration_ms: 0,
17272 folded: false,
17273 };
17274 let (e2e, retried, pids) = run_e2e_with_retry(
17275 &mut runner.state,
17276 &["sh".into(), "-c".into()],
17277 &[format!("{probe} && printf initial")],
17278 &repo,
17279 Duration::from_secs(10),
17280 "initial",
17281 )
17282 .await;
17283 assert!(!retried && pids.is_empty());
17284 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17285 assert_eq!(e2e[0].output_tail, "initial");
17286 let gate = runner.run_gate(&winner).await.unwrap();
17287 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17288 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17291 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17292 passed: false,
17293 outcomes: Vec::new(),
17294 });
17295 runner.state.save().unwrap();
17296 let mut resumed = RunState::load(&runner.state.id).unwrap();
17297 let wait = crate::repo_lock::WaitPolicy::default();
17298 assert!(matches!(
17299 check_gate_repair_e2e(&mut resumed, &winner, &wait)
17300 .await
17301 .unwrap(),
17302 GateFix::Retry
17303 ));
17304 let saved = RunState::load(&resumed.id).unwrap();
17305 let repair = saved.gate_repair_e2e.unwrap();
17306 assert!(repair.passed);
17307 assert_eq!(repair.outcomes.len(), 1);
17308 assert!(repair.outcomes[0].ok());
17309 let landing = run_landing_gate(&mut resumed, &winner, &wait)
17310 .await
17311 .unwrap();
17312 assert_eq!(landing.len(), 1);
17313 assert!(landing[0].ok(), "{landing:?}");
17314 assert_eq!(
17315 std::fs::read_to_string(repo.join("phases")).unwrap(),
17316 "pre\ngate\ne2e\npre\ngate\n"
17317 );
17318 assert_worker();
17319 }
17320
17321 #[cfg(unix)]
17322 #[tokio::test]
17323 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
17324 crate::run::pin_test_home();
17325 let tmp = tempfile::tempdir().unwrap();
17326 let mut config = Config::default();
17327 config.disk.min_free_bytes = 0;
17328 let mut state = RunState::new(
17329 tmp.path().to_owned(),
17330 "main".into(),
17331 "base".into(),
17332 "verification evidence".into(),
17333 config,
17334 );
17335 let (outcomes, pids) = run_commands(
17336 &mut state,
17337 "gate",
17338 "gate",
17339 1,
17340 &["sh".into(), "-c".into()],
17341 &[
17342 "printf before-timeout; exec sleep 30".into(),
17343 "kill -TERM $$".into(),
17344 "exit 7".into(),
17345 "printf success".into(),
17346 ],
17347 tmp.path(),
17348 Duration::from_secs(1),
17349 )
17350 .await;
17351 wait_for_timed_out_children_to_die(&pids).await;
17352 assert_eq!(
17353 pids.len(),
17354 1,
17355 "unleased timeout retains its death-check PID"
17356 );
17357 assert_eq!(outcomes.len(), 4);
17358 assert_eq!(outcomes[0].code, None);
17359 assert!(outcomes[0].output_tail.contains("before-timeout"));
17360 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
17361 assert_eq!(outcomes[1].code, None);
17362 assert!(outcomes[1].output_tail.contains("command terminated:"));
17363 assert!(!outcomes[1].output_tail.contains("timed out"));
17364 assert_eq!(outcomes[2].code, Some(7));
17365 assert!(!outcomes[2].output_tail.contains("timed out"));
17366 assert_eq!(outcomes[3].code, Some(0));
17367 assert_eq!(outcomes[3].output_tail, "success");
17368 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
17369 }
17370
17371 #[tokio::test]
17381 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
17382 crate::run::pin_test_home();
17383
17384 let tmp = tempfile::tempdir().expect("tempdir");
17385 let repo = tmp.path().join("repo");
17386 std::fs::create_dir_all(&repo).unwrap();
17387 init_repo(&repo);
17388
17389 let mut config = Config::default();
17390 config.verify.gate = vec![
17391 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
17392 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
17393 .to_owned(),
17394 ];
17395
17396 let mut state = RunState::new(
17397 repo.clone(),
17398 "main".to_owned(),
17399 "deadbeef".to_owned(),
17400 "task".to_owned(),
17401 config,
17402 );
17403 let run_id = state.id.clone();
17404 state.candidates = vec![Candidate {
17405 index: 0,
17406 label: 'A',
17407 agent: "alpha".to_owned(),
17408 branch: "does-not-exist".to_owned(),
17409 worktree: repo.clone(),
17410 summary: String::new(),
17411 stat: String::new(),
17412 files: 0,
17413 commits: 0,
17414 empty: false,
17415 failed: None,
17416 verified_noop: None,
17417 duration_ms: 0,
17418 folded: false,
17419 }];
17420 state.tally = Some(Tally {
17421 first_choice: BTreeMap::from([('A', 1)]),
17422 borda: BTreeMap::new(),
17423 winner: 'A',
17424 rankings: 1,
17425 unanimous_initial: true,
17426 deliberated: false,
17427 changed_votes: 0,
17428 unanimous_final: true,
17429 tie_break: None,
17430 judges: 0,
17431 present: 0,
17432 quorum: 0,
17433 met_quorum: true,
17434 uncontested: Some("only candidate A produced a change".to_owned()),
17435 });
17436 state.reviews = vec![ReviewRound {
17437 round: 1,
17438 head: "deadbeef".to_owned(),
17439 verified_head: None,
17440 verified_at: None,
17441 reviews: Vec::new(),
17442 e2e: Vec::new(),
17443 fix: None,
17444 blocking: 0,
17445 answered: 0,
17446 expected: 0,
17447 clean: true,
17448 verify_retried: false,
17449 e2e_deferred: false,
17450 e2e_defer_reason: None,
17451 progressed: false,
17452 vote_split: false,
17453 reconsideration: Vec::new(),
17454 verdict: None,
17455 }];
17456
17457 let mut runner = Runner {
17458 state,
17459 roles: ResolvedRoles {
17460 implementers: Vec::new(),
17461 judges: Vec::new(),
17462 reviewers: Vec::new(),
17463 fixer: None,
17464 conductor: conductor(),
17465 implementer_roster: Vec::new(),
17466 judge_roster: Vec::new(),
17467 reviewer_roster: Vec::new(),
17468 },
17469 sem: Arc::new(Semaphore::new(1)),
17470 pause: Pause::new(),
17471 interrupt: Pause::new(),
17472 };
17473
17474 let started_marker = repo.join("started.marker");
17475 let release_marker = repo.join("release.marker");
17476 let poller = tokio::spawn(async move {
17477 for _ in 0..100 {
17482 if started_marker.exists()
17483 && let Ok(s) = crate::run::RunState::load(&run_id)
17484 && let Some(a) = s.active.get("gate")
17485 {
17486 std::fs::write(&release_marker, b"go").expect("release marker");
17487 return Some(a.clone());
17488 }
17489 tokio::time::sleep(Duration::from_millis(50)).await;
17490 }
17491 None
17492 });
17493
17494 runner.gate().await.expect("gate");
17495 let captured = poller.await.expect("poller task");
17496 let captured = captured.expect(
17497 "the poller never saw a `gate` task entry in run.json while the command was \
17498 still blocked on its own release marker",
17499 );
17500
17501 assert_eq!(captured.task.as_deref(), Some("gate"));
17502 assert_eq!(captured.node, "gate");
17503 assert_eq!(captured.index, Some(1));
17504 assert_eq!(captured.total, Some(1));
17505 assert!(
17506 captured
17507 .command
17508 .as_deref()
17509 .is_some_and(|c| c.contains("started.marker")),
17510 "{captured:?}"
17511 );
17512
17513 assert!(
17514 runner.state.active.is_empty(),
17515 "the entry must be cleared once the command actually finished: {:?}",
17516 runner.state.active
17517 );
17518 assert!(runner.state.gate_ran);
17519 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
17520 }
17521
17522 #[tokio::test]
17525 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
17526 use crate::verdict::{Finding, ReviewVote, Severity};
17527 crate::run::pin_test_home();
17528 let tmp = tempfile::tempdir().expect("tempdir");
17529 let repo = tmp.path().join("repo");
17530 std::fs::create_dir_all(&repo).unwrap();
17531 init_repo(&repo);
17532
17533 for (severity, vote, expect) in [
17534 (Severity::Major, ReviewVote::Reject, true),
17535 (Severity::Minor, ReviewVote::Reject, false),
17536 (Severity::Major, ReviewVote::Approve, false),
17537 ] {
17538 let mut round = review_round(false, 1, 1, 1, false, true);
17539 round.reviews = vec![ReviewRecord {
17540 reviewer: 1,
17541 agent: "alpha".to_owned(),
17542 summary: String::new(),
17543 findings: vec![Finding {
17544 id: "R1-1-1".to_owned(),
17545 severity,
17546 file: None,
17547 line: None,
17548 title: "t".to_owned(),
17549 detail: String::new(),
17550 }],
17551 vote: Some(vote),
17552 failed: None,
17553 duration_ms: 0,
17554 attempts: 0,
17555 }];
17556 let mut state = RunState::new(
17557 repo.clone(),
17558 "main".to_owned(),
17559 "deadbeef".to_owned(),
17560 "task".to_owned(),
17561 Config::default(),
17562 );
17563 state.reviews = vec![round];
17564 let mut runner = Runner {
17565 state,
17566 roles: ResolvedRoles {
17567 implementers: Vec::new(),
17568 judges: Vec::new(),
17569 reviewers: Vec::new(),
17570 fixer: None,
17571 conductor: conductor(),
17572 implementer_roster: Vec::new(),
17573 judge_roster: Vec::new(),
17574 reviewer_roster: Vec::new(),
17575 },
17576 sem: Arc::new(Semaphore::new(1)),
17577 pause: Pause::new(),
17578 interrupt: Pause::new(),
17579 };
17580 let shell = runner.state.config.shell();
17581 runner
17582 .stop_reviewing("round budget spent", &shell, &repo)
17583 .await
17584 .expect("stop_reviewing");
17585 assert_eq!(runner.state.status, RunStatus::Gating);
17586 assert_eq!(
17587 runner.state.contested_handoff.is_some(),
17588 expect,
17589 "{severity:?} + {vote:?}"
17590 );
17591 }
17592 }
17593
17594 #[tokio::test]
17607 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
17608 crate::run::pin_test_home();
17609 let home = crate::run::home();
17610
17611 let tmp = tempfile::tempdir().expect("tempdir");
17612 let repo = tmp.path().join("repo");
17613 std::fs::create_dir_all(&repo).unwrap();
17614 init_repo(&repo);
17615 let head = crate::git::rev_parse(&repo, "HEAD")
17616 .await
17617 .expect("rev-parse");
17618 let cache_dir = tmp.path().join("target");
17621
17622 let mut config = Config::default();
17623 config.verify.e2e = vec![format!(
17624 "CARGO_TARGET_DIR='{}' test -f README.md",
17625 cache_dir.display()
17626 )];
17627 config.graph.review_rounds = 1;
17628 config.graph.timeout_verify = Some(2);
17631 config.graph.timeout_resource_wait = 2;
17632
17633 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17634 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17635 .expect("no io error acquiring directly")
17636 {
17637 crate::cache::AcquireOutcome::Acquired(g) => g,
17638 crate::cache::AcquireOutcome::Busy(b) => {
17639 panic!("expected the direct acquire to win the lease first: {b:?}")
17640 }
17641 };
17642
17643 let mut state = RunState::new(
17644 repo.clone(),
17645 "main".to_owned(),
17646 head.clone(),
17647 "task".to_owned(),
17648 config,
17649 );
17650 state.candidates = vec![Candidate {
17651 index: 0,
17652 label: 'A',
17653 agent: "alpha".to_owned(),
17654 branch: "does-not-exist".to_owned(),
17655 worktree: repo.clone(),
17656 summary: String::new(),
17657 stat: String::new(),
17658 files: 0,
17659 commits: 0,
17660 empty: false,
17661 failed: None,
17662 verified_noop: None,
17663 duration_ms: 0,
17664 folded: false,
17665 }];
17666 state.tally = Some(Tally {
17667 first_choice: BTreeMap::from([('A', 1)]),
17668 borda: BTreeMap::new(),
17669 winner: 'A',
17670 rankings: 1,
17671 unanimous_initial: true,
17672 deliberated: false,
17673 changed_votes: 0,
17674 unanimous_final: true,
17675 tie_break: None,
17676 judges: 0,
17677 present: 0,
17678 quorum: 0,
17679 met_quorum: true,
17680 uncontested: Some("only candidate A produced a change".to_owned()),
17681 });
17682 state.reviews = vec![ReviewRound {
17686 round: 1,
17687 head: head.clone(),
17688 verified_head: None,
17689 verified_at: None,
17690 reviews: Vec::new(),
17691 e2e: Vec::new(),
17692 fix: None,
17693 blocking: 1,
17694 answered: 1,
17695 expected: 1,
17696 clean: false,
17697 verify_retried: false,
17698 e2e_deferred: true,
17699 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
17700 progressed: false,
17701 vote_split: false,
17702 reconsideration: Vec::new(),
17703 verdict: None,
17704 }];
17705
17706 let mut runner = Runner {
17707 state,
17708 roles: ResolvedRoles {
17709 implementers: Vec::new(),
17710 judges: Vec::new(),
17711 reviewers: Vec::new(),
17712 fixer: None,
17713 conductor: conductor(),
17714 implementer_roster: Vec::new(),
17715 judge_roster: Vec::new(),
17716 reviewer_roster: Vec::new(),
17717 },
17718 sem: Arc::new(Semaphore::new(1)),
17719 pause: Pause::new(),
17720 interrupt: Pause::new(),
17721 };
17722
17723 let shell = runner.state.config.shell();
17724 runner
17725 .stop_reviewing("round budget spent", &shell, &repo)
17726 .await
17727 .expect("stop_reviewing");
17728
17729 let last = runner.state.reviews.last().expect("round record");
17730 assert_eq!(
17731 last.e2e_status(),
17732 E2eStatus::ResourceBlocked,
17733 "the shared cache is still held; the attempt must read as blocked, not deferred or \
17734 failed: {last:?}"
17735 );
17736 assert_eq!(
17737 last.verified_head.as_deref(),
17738 Some(head.as_str()),
17739 "which commit this attempt targeted is known even though nothing finished checking \
17740 it"
17741 );
17742 let first_attempt_at = last
17743 .verified_at
17744 .expect("when this attempt ran is known too");
17745 assert_ne!(
17746 runner.state.status,
17747 RunStatus::Blocked,
17748 "contention is evidence about the machine, not the patch — it must not settle the \
17749 run as blocked: {:?}",
17750 runner.state.status
17751 );
17752 assert!(
17753 !runner
17754 .state
17755 .events
17756 .iter()
17757 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
17758 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
17759 runner.state.events
17760 );
17761
17762 runner
17767 .stop_reviewing("round budget spent", &shell, &repo)
17768 .await
17769 .expect("stop_reviewing retry");
17770 assert_eq!(
17771 runner.state.reviews.len(),
17772 1,
17773 "no new round was started: {:?}",
17774 runner.state.reviews
17775 );
17776 let last = runner.state.reviews.last().expect("round record");
17777 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
17778 assert!(
17779 last.verified_at.expect("still known") > first_attempt_at,
17780 "a second reentry must be a fresh attempt, not a stale copy of the first"
17781 );
17782 assert_ne!(runner.state.status, RunStatus::Blocked);
17783
17784 held.release();
17785 }
17786
17787 #[cfg(unix)]
17788 #[tokio::test]
17789 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
17790 ask_test_home();
17791 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
17792 let tmp = tempfile::tempdir().unwrap();
17793 let repo = tmp.path().join("repo");
17794 std::fs::create_dir(&repo).unwrap();
17795 init_repo(&repo);
17796 let run_git = |args: &[&str]| {
17797 let out = std::process::Command::new("git")
17798 .args(args)
17799 .current_dir(&repo)
17800 .output()
17801 .unwrap();
17802 assert!(
17803 out.status.success(),
17804 "{args:?}: {}",
17805 String::from_utf8_lossy(&out.stderr)
17806 );
17807 };
17808 let origin = tmp.path().join("origin.git");
17809 run_git(&["init", "--bare", origin.to_str().unwrap()]);
17810 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17811 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17812 run_git(&["add", "spec.rs"]);
17813 run_git(&["commit", "-m", "initial spec"]);
17814 run_git(&["push", "origin", "main"]);
17815 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17816 run_git(&["checkout", "-b", "fix/review"]);
17817 std::fs::write(
17818 repo.join("check.rs"),
17819 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17820 )
17821 .unwrap();
17822 run_git(&["add", "check.rs"]);
17823 run_git(&["commit", "-m", "use spec"]);
17824 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17825 run_git(&["checkout", "main"]);
17826 std::fs::write(
17827 repo.join("spec.rs"),
17828 "struct AgentSpec { max_turn_secs: u64 }\n",
17829 )
17830 .unwrap();
17831 run_git(&["add", "spec.rs"]);
17832 run_git(&["commit", "-m", "extend spec"]);
17833 run_git(&["push", "origin", "main"]);
17834 run_git(&["checkout", "fix/review"]);
17835
17836 let mut runner = runner_at(RunStatus::Reviewing);
17837 runner.state.repo = repo.clone();
17838 runner.state.base_commit = base;
17839 runner.state.pr_head = Some("fix/review".into());
17840 runner.state.config.graph.review_rounds = max_rounds;
17841 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17842 runner.state.config.graph.sessions = false;
17843 runner.state.config.disk.min_free_bytes = 0;
17844 runner.state.config.verify.e2e =
17845 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17846 runner.state.candidates = vec![Candidate {
17847 index: 0,
17848 label: 'A',
17849 agent: "fixture".into(),
17850 branch: "fix/review".into(),
17851 worktree: repo.clone(),
17852 summary: String::new(),
17853 stat: String::new(),
17854 files: 1,
17855 commits: 1,
17856 empty: false,
17857 failed: None,
17858 verified_noop: None,
17859 duration_ms: 0,
17860 folded: false,
17861 }];
17862 runner.state.tally = Some(Tally {
17863 first_choice: BTreeMap::new(),
17864 borda: BTreeMap::new(),
17865 winner: 'A',
17866 rankings: 0,
17867 unanimous_initial: true,
17868 deliberated: false,
17869 changed_votes: 0,
17870 unanimous_final: true,
17871 tie_break: None,
17872 judges: 0,
17873 present: 0,
17874 quorum: 0,
17875 met_quorum: true,
17876 uncontested: Some("review-only".into()),
17877 });
17878 let mut last = review_round(false, 0, 1, 1, false, false);
17879 last.head = reviewed.clone();
17880 last.e2e_deferred = true;
17881 last.e2e[0].resource_blocked = true;
17882 last.e2e[0].output_tail = "waiting for the shared build cache".into();
17883 if max_rounds == 1 {
17884 last.e2e.clear();
17885 }
17886 runner.state.reviews = vec![last];
17887 let calls = tmp.path().join("fix-calls");
17888 let mut fixer = spec("repair");
17889 fixer
17890 .env
17891 .insert("FIX_CALLS".into(), calls.display().to_string());
17892 fixer.command = vec!["sh".into(), "-c".into(), r#"
17893 echo fix >> "$FIX_CALLS"
17894 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
17895 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
17896 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
17897 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
17898 "#.into()];
17899 let mut reviewer = spec("reviewer");
17900 reviewer.command = vec![
17901 "sh".into(),
17902 "-c".into(),
17903 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
17904 .into(),
17905 ];
17906 runner.roles.fixer = Some(vec![fixer]);
17907 runner.roles.reviewers = vec![reviewer];
17908
17909 runner.sync_to_base().await.unwrap();
17910 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
17911 assert_ne!(rebased, reviewed);
17912 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
17913 if reentry {
17914 let shell = runner.state.config.shell();
17917 runner
17918 .stop_unresolved_review("deferred verification", &shell, &repo)
17919 .await
17920 .unwrap();
17921 assert!(runner.catchup_repairable());
17922 assert!(!calls.exists());
17923 runner.state = RunState::load(&runner.state.id).unwrap();
17924 assert!(runner.catchup_repairable());
17925 }
17926 runner.review_loop().await.unwrap();
17927 let first = &runner.state.reviews[0];
17928 assert_eq!(first.head, reviewed);
17929 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
17930 assert!(
17931 first.verify_retried,
17932 "a reproduced build failure gets exactly one retry: {:?}",
17933 first.e2e
17934 );
17935 assert!(first.e2e[0].output_tail.contains("E0063"));
17936 assert!(!runner.catchup_repairable());
17937 if max_rounds == 2 {
17938 assert_eq!(
17939 runner.state.status,
17940 RunStatus::Gating,
17941 "{}",
17942 crate::report::run(&runner.state)
17943 );
17944 assert_eq!(runner.state.reviews.len(), 2);
17945 assert!(first.fix.as_ref().unwrap().committed);
17946 assert!(runner.state.reviews[1].clean);
17947 assert_eq!(
17948 runner
17949 .state
17950 .reviews
17951 .iter()
17952 .filter(|r| r.fix.is_some())
17953 .count(),
17954 1
17955 );
17956 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
17957 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
17958 } else {
17959 assert_eq!(runner.state.status, RunStatus::Blocked);
17960 assert!(first.fix.is_none());
17961 assert!(!calls.exists());
17962 let recovery = crate::recovery::assess(
17963 "prior",
17964 Some(&runner.state),
17965 &runner.landing_base(),
17966 crate::recovery::Relation::Diverged,
17967 );
17968 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
17969 let reasons = recovery.reasons.join("\n");
17970 assert!(
17971 reasons.contains(&reviewed) && reasons.contains(&rebased),
17972 "{reasons}"
17973 );
17974 assert!(
17975 reasons.contains("base sync to")
17976 && reasons.contains("additional review is required"),
17977 "{reasons}"
17978 );
17979 }
17980 }
17981 }
17982
17983 #[tokio::test]
17995 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
17996 crate::run::pin_test_home();
17997 let home = crate::run::home();
17998
17999 let tmp = tempfile::tempdir().expect("tempdir");
18000 let repo = tmp.path().join("repo");
18001 std::fs::create_dir_all(&repo).unwrap();
18002 init_repo(&repo);
18003 let head = crate::git::rev_parse(&repo, "HEAD")
18004 .await
18005 .expect("rev-parse");
18006 let cache_dir = tmp.path().join("target");
18007
18008 let mut config = Config::default();
18009 config.verify.e2e = vec![format!(
18010 "CARGO_TARGET_DIR='{}' test -f README.md",
18011 cache_dir.display()
18012 )];
18013 config.graph.review_rounds = 1;
18014 config.graph.timeout_verify = Some(2);
18015 config.graph.timeout_resource_wait = 2;
18016
18017 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18018 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18019 .expect("no io error acquiring directly")
18020 {
18021 crate::cache::AcquireOutcome::Acquired(g) => g,
18022 crate::cache::AcquireOutcome::Busy(b) => {
18023 panic!("expected the direct acquire to win the lease first: {b:?}")
18024 }
18025 };
18026
18027 let mut state = RunState::new(
18028 repo.clone(),
18029 "main".to_owned(),
18030 head.clone(),
18031 "task".to_owned(),
18032 config,
18033 );
18034 state.candidates = vec![Candidate {
18035 index: 0,
18036 label: 'A',
18037 agent: "alpha".to_owned(),
18038 branch: "does-not-exist".to_owned(),
18039 worktree: repo.clone(),
18040 summary: String::new(),
18041 stat: String::new(),
18042 files: 0,
18043 commits: 0,
18044 empty: false,
18045 failed: None,
18046 verified_noop: None,
18047 duration_ms: 0,
18048 folded: false,
18049 }];
18050 state.tally = Some(Tally {
18051 first_choice: BTreeMap::from([('A', 1)]),
18052 borda: BTreeMap::new(),
18053 winner: 'A',
18054 rankings: 1,
18055 unanimous_initial: true,
18056 deliberated: false,
18057 changed_votes: 0,
18058 unanimous_final: true,
18059 tie_break: None,
18060 judges: 0,
18061 present: 0,
18062 quorum: 0,
18063 met_quorum: true,
18064 uncontested: Some("only candidate A produced a change".to_owned()),
18065 });
18066 state.reviews = vec![ReviewRound {
18070 round: 1,
18071 head: head.clone(),
18072 verified_head: Some(head.clone()),
18073 verified_at: Some(jiff::Timestamp::now()),
18074 reviews: Vec::new(),
18075 e2e: vec![CommandOutcome {
18076 command: format!(
18077 "CARGO_TARGET_DIR='{}' test -f README.md",
18078 cache_dir.display()
18079 ),
18080 code: None,
18081 output_tail: "waiting for the shared build cache".to_owned(),
18082 duration_ms: 0,
18083 resource_blocked: true,
18084 }],
18085 fix: None,
18086 blocking: 1,
18087 answered: 1,
18088 expected: 1,
18089 clean: false,
18090 verify_retried: false,
18091 e2e_deferred: false,
18092 e2e_defer_reason: None,
18093 progressed: false,
18094 vote_split: false,
18095 reconsideration: Vec::new(),
18096 verdict: None,
18097 }];
18098
18099 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18100 let mut runner = Runner {
18101 state,
18102 roles: ResolvedRoles {
18103 implementers: Vec::new(),
18104 judges: Vec::new(),
18105 reviewers: Vec::new(),
18106 fixer: None,
18107 conductor: conductor(),
18108 implementer_roster: Vec::new(),
18109 judge_roster: Vec::new(),
18110 reviewer_roster: Vec::new(),
18111 },
18112 sem: Arc::new(Semaphore::new(1)),
18113 pause: Pause::new(),
18114 interrupt: Pause::new(),
18115 };
18116
18117 runner.review_loop().await.expect("review_loop");
18122
18123 assert_eq!(
18124 runner.state.reviews.len(),
18125 1,
18126 "no new round was started on top of the unresolved one: {:?}",
18127 runner.state.reviews
18128 );
18129 let last = &runner.state.reviews[0];
18130 assert_eq!(
18131 last.e2e_status(),
18132 E2eStatus::ResourceBlocked,
18133 "still contended: {last:?}"
18134 );
18135 assert!(
18136 last.verified_at.expect("still known") > first_attempt_at,
18137 "review_loop must have actually retried the check, not left it exactly as found"
18138 );
18139 assert_ne!(
18140 runner.state.status,
18141 RunStatus::Blocked,
18142 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18143 runner.state.status
18144 );
18145
18146 held.release();
18147 }
18148
18149 #[tokio::test]
18150 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18151 crate::run::pin_test_home();
18152 for status in [RunStatus::Blocked, RunStatus::Gating] {
18153 let mut runner = runner_at(status);
18154 runner.state.config.graph.land = true;
18155 runner.state.gate_ran = true;
18156 runner.state.merge = Some(MergeOutcome {
18157 mode: MergeMode::Pr,
18158 ok: false,
18159 detail: "the fixer produced no commit".into(),
18160 empty: false,
18161 });
18162 let url = "https://example.invalid/o/r/pull/1";
18163 runner.state.pr = Some(crate::run::PrRecord {
18164 url: url.into(),
18165 number: 1,
18166 state: "open".into(),
18167 checks: "red".into(),
18168 round: 1,
18169 rounds: 1,
18170 red_at_merge: Vec::new(),
18171 });
18172 runner.state.save().unwrap();
18173 runner.execute_graph().await.unwrap();
18177 assert_eq!(runner.state.status, RunStatus::Blocked);
18178 assert!(runner.state.land_sync_pending);
18179 assert!(runner.state.events.iter().any(|event|
18180 event.node == "land" && event.message == format!("watching {url}")));
18181 assert_eq!(
18182 runner.state.merge.as_ref().unwrap().detail,
18183 "the fixer produced no commit"
18184 );
18185 }
18186 }
18187
18188 #[tokio::test]
18189 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18190 crate::run::pin_test_home();
18191 let tmp = tempfile::tempdir().expect("tempdir");
18192 let repo = tmp.path().join("repo");
18193 std::fs::create_dir_all(&repo).unwrap();
18194 init_repo(&repo);
18195
18196 let mut config = Config::default();
18197 config.merge.mode = MergeMode::Pr;
18198 config.graph.land = true;
18199 config.graph.land_approval = false;
18200
18201 let mut state = RunState::new(
18202 repo.clone(),
18203 "main".to_owned(),
18204 "deadbeef".to_owned(),
18205 "task".to_owned(),
18206 config,
18207 );
18208 state.candidates = vec![Candidate {
18209 index: 0,
18210 label: 'A',
18211 agent: "alpha".to_owned(),
18212 branch: "does-not-exist".to_owned(),
18213 worktree: repo.clone(),
18214 summary: String::new(),
18215 stat: String::new(),
18216 files: 0,
18217 commits: 0,
18218 empty: false,
18219 failed: None,
18220 verified_noop: None,
18221 duration_ms: 0,
18222 folded: false,
18223 }];
18224 state.tally = Some(Tally {
18225 first_choice: BTreeMap::from([('A', 1)]),
18226 borda: BTreeMap::new(),
18227 winner: 'A',
18228 rankings: 1,
18229 unanimous_initial: true,
18230 deliberated: false,
18231 changed_votes: 0,
18232 unanimous_final: true,
18233 tie_break: None,
18234 judges: 0,
18235 present: 0,
18236 quorum: 0,
18237 met_quorum: true,
18238 uncontested: Some("only candidate A produced a change".to_owned()),
18239 });
18240 state.reviews = vec![ReviewRound {
18241 round: 1,
18242 head: "deadbeef".to_owned(),
18243 verified_head: None,
18244 verified_at: None,
18245 reviews: Vec::new(),
18246 e2e: Vec::new(),
18247 fix: None,
18248 blocking: 0,
18249 answered: 0,
18250 expected: 0,
18251 clean: true,
18252 verify_retried: false,
18253 e2e_deferred: false,
18254 e2e_defer_reason: None,
18255 progressed: false,
18256 vote_split: false,
18257 reconsideration: Vec::new(),
18258 verdict: None,
18259 }];
18260 state.gate = vec![CommandOutcome {
18261 command: "test".to_owned(),
18262 code: Some(0),
18263 output_tail: String::new(),
18264 duration_ms: 0,
18265 resource_blocked: false,
18266 }];
18267 state.gate_ran = true;
18268 state.status = RunStatus::Landing;
18272 state.merge = Some(MergeOutcome {
18273 mode: MergeMode::Pr,
18274 ok: true,
18275 detail: "https://example.invalid/x/y/pull/1".to_owned(),
18276 empty: false,
18277 });
18278
18279 ask_test_home();
18283 state.save().expect("save landing run");
18286 let store = ask::Questions::open();
18287 let q = ask_open_question(&store, &state.id);
18288
18289 let mut runner = Runner {
18290 state,
18291 roles: ResolvedRoles {
18292 implementers: Vec::new(),
18293 judges: Vec::new(),
18294 reviewers: Vec::new(),
18295 fixer: None,
18296 conductor: conductor(),
18297 implementer_roster: Vec::new(),
18298 judge_roster: Vec::new(),
18299 reviewer_roster: Vec::new(),
18300 },
18301 sem: Arc::new(Semaphore::new(1)),
18302 pause: Pause::new(),
18303 interrupt: Pause::new(),
18304 };
18305
18306 runner.execute().await.expect("execute");
18311
18312 assert_eq!(
18313 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
18314 Some("https://example.invalid/x/y/pull/1"),
18315 "reentry must not push again or open a second pull request over the \
18316 one `land` is already watching"
18317 );
18318 assert_ne!(
18319 runner.state.status,
18320 RunStatus::Landing,
18321 "land could not actually reach the fake pull request, so it must \
18322 have given up rather than left the run silently parked forever"
18323 );
18324 assert_eq!(runner.state.status, RunStatus::Blocked);
18328 assert!(
18329 store.get(&q.id).unwrap().status.open(),
18330 "Blocked is still alive; settle_questions must have been a no-op here"
18331 );
18332 }
18333
18334 fn state_with_round(round: ReviewRound) -> RunState {
18335 let mut s = RunState::new(
18336 PathBuf::from("/repo"),
18337 "main".to_owned(),
18338 "abc1234".to_owned(),
18339 "add retries".to_owned(),
18340 Config::default(),
18341 );
18342 s.reviews = vec![round];
18343 s
18344 }
18345
18346 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
18347 crate::verdict::Finding {
18348 id: id.to_owned(),
18349 severity,
18350 file: None,
18351 line: None,
18352 title: title.to_owned(),
18353 detail: String::new(),
18354 }
18355 }
18356
18357 #[test]
18358 fn pr_body_names_open_findings_and_declined_ones() {
18359 let round = ReviewRound {
18360 round: 2,
18361 head: "deadbee".to_owned(),
18362 verified_head: None,
18363 verified_at: None,
18364 reviews: vec![ReviewRecord {
18365 attempts: 0,
18366 reviewer: 1,
18367 agent: "alpha".to_owned(),
18368 summary: String::new(),
18369 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
18370 vote: None,
18371 failed: None,
18372 duration_ms: 0,
18373 }],
18374 e2e: vec![CommandOutcome {
18375 command: "cargo test".to_owned(),
18376 code: Some(0),
18377 output_tail: String::new(),
18378 duration_ms: 0,
18379 resource_blocked: false,
18380 }],
18381 verify_retried: false,
18382 e2e_deferred: false,
18383 e2e_defer_reason: None,
18384 fix: Some(FixRecord {
18385 agent: "alpha".to_owned(),
18386 addressed: Vec::new(),
18387 rejected: vec![crate::verdict::Rejection {
18388 id: "R1-1-1".to_owned(),
18389 why: "not reachable from any caller".to_owned(),
18390 }],
18391 notes: String::new(),
18392 committed: true,
18393 failed: None,
18394 duration_ms: 0,
18395 continuation: None,
18396 }),
18397 blocking: 0,
18398 answered: 1,
18399 expected: 1,
18400 clean: false,
18401 progressed: true,
18402 vote_split: false,
18403 reconsideration: Vec::new(),
18404 verdict: None,
18405 };
18406 let state = state_with_round(round);
18407 let body = pr_message(&state, 'A').body;
18408
18409 assert!(body.contains("add retries"), "the task must still be there");
18410 assert!(body.contains("R2-1-1"), "{body}");
18411 assert!(!body.contains("unused import"), "{body}");
18412 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
18413 assert!(
18414 !body.contains("not reachable from any caller"),
18415 "the reason it was declined: {body}"
18416 );
18417 }
18418
18419 #[test]
18420 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
18421 let round = ReviewRound {
18422 round: 1,
18423 head: "deadbee".to_owned(),
18424 verified_head: None,
18425 verified_at: None,
18426 reviews: vec![ReviewRecord {
18427 attempts: 0,
18428 reviewer: 1,
18429 agent: "alpha".to_owned(),
18430 summary: String::new(),
18431 findings: Vec::new(),
18432 vote: None,
18433 failed: None,
18434 duration_ms: 0,
18435 }],
18436 e2e: Vec::new(),
18437 verify_retried: false,
18438 e2e_deferred: false,
18439 e2e_defer_reason: None,
18440 fix: None,
18441 blocking: 0,
18442 answered: 1,
18443 expected: 1,
18444 clean: true,
18445 progressed: false,
18446 vote_split: false,
18447 reconsideration: Vec::new(),
18448 verdict: None,
18449 };
18450 let state = state_with_round(round);
18451 let body = pr_message(&state, 'A').body;
18452 assert!(!body.contains("Open review findings"), "{body}");
18453 assert!(!body.contains("Declined"), "{body}");
18454 }
18455
18456 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
18457 let mut state = RunState::new(
18458 PathBuf::from("/repo"),
18459 "main".to_owned(),
18460 "abc1234".to_owned(),
18461 instruction.to_owned(),
18462 Config::default(),
18463 );
18464 state.candidates.push(Candidate {
18465 index: 0,
18466 label: 'A',
18467 agent: "alpha".to_owned(),
18468 branch: "magi/x/A".to_owned(),
18469 worktree: PathBuf::from("/wt"),
18470 summary: summary.to_owned(),
18471 stat: String::new(),
18472 files: 1,
18473 commits: 1,
18474 empty: false,
18475 failed: None,
18476 verified_noop: None,
18477 folded: false,
18478 duration_ms: 0,
18479 });
18480 state
18481 }
18482
18483 fn review_state(subjects: &[&str]) -> RunState {
18484 let mut state = state_with_summary(
18485 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
18486 "",
18487 );
18488 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18489 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
18490 state
18491 }
18492
18493 #[tokio::test]
18496 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
18497 ask_test_home();
18498 let tmp = tempfile::tempdir().unwrap();
18499 let repo = tmp.path().join("repo");
18500 std::fs::create_dir_all(&repo).unwrap();
18501 init_repo(&repo);
18502 let origin = tmp.path().join("origin.git");
18503 let g = |dir: &Path, args: &[&str]| {
18504 let out = std::process::Command::new("git")
18505 .args(args)
18506 .current_dir(dir)
18507 .quiet()
18508 .output()
18509 .expect("spawn git");
18510 assert!(
18511 out.status.success(),
18512 "git {args:?}: {}",
18513 String::from_utf8_lossy(&out.stderr)
18514 );
18515 };
18516 g(
18517 tmp.path(),
18518 &[
18519 "clone",
18520 "--bare",
18521 "-q",
18522 repo.to_str().unwrap(),
18523 origin.to_str().unwrap(),
18524 ],
18525 );
18526 g(
18527 &repo,
18528 &["remote", "add", "origin", origin.to_str().unwrap()],
18529 );
18530 let c1 = git::rev_parse(&repo, "main").await.unwrap();
18531
18532 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
18534 g(&repo, &["add", "-A"]);
18535 g(
18536 &repo,
18537 &["commit", "-q", "-m", "chore(deps): update a crate"],
18538 );
18539 g(&repo, &["push", "-q", "origin", "main"]);
18540 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
18541 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
18542 g(&repo, &["add", "-A"]);
18543 g(
18544 &repo,
18545 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
18546 );
18547 g(&repo, &["checkout", "-q", "main"]);
18548
18549 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
18550 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
18551 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
18553 assert_eq!(fallback, c1);
18554
18555 let mut state = review_state(&[
18556 "chore(deps): update a crate",
18557 "fix(daemon): apply a chosen action",
18558 ]);
18559 state.repo = repo.clone();
18560 state.base_branch = "main".to_owned();
18561 state.base_commit = c1;
18562 refresh_reviewed_commits(&mut state, "feat/own").await;
18563 assert_eq!(
18564 state.reviewed_commits,
18565 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
18566 );
18567 assert_eq!(
18568 review_title(&state).as_deref(),
18569 Some("fix(daemon): apply a chosen action")
18570 );
18571 assert_eq!(
18572 leaked_subjects(&state, "feat/own").await,
18573 Some(vec!["chore(deps): update a crate".to_owned()])
18574 );
18575 state.config.merge.remote = "nowhere".to_owned();
18578 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
18579 }
18580
18581 struct SummaryFixture {
18585 _tmp: tempfile::TempDir,
18586 repo: PathBuf,
18587 reply: PathBuf,
18588 runner: Runner,
18589 }
18590
18591 fn summary_fixture() -> SummaryFixture {
18592 ask_test_home();
18593 let tmp = tempfile::tempdir().unwrap();
18594 let repo = tmp.path().join("repo");
18595 std::fs::create_dir_all(&repo).unwrap();
18596 init_repo(&repo);
18597 let origin = tmp.path().join("origin.git");
18598 let g = |args: &[&str]| {
18599 let out = std::process::Command::new("git")
18600 .args(args)
18601 .current_dir(&repo)
18602 .quiet()
18603 .output()
18604 .expect("spawn git");
18605 assert!(
18606 out.status.success(),
18607 "git {args:?}: {}",
18608 String::from_utf8_lossy(&out.stderr)
18609 );
18610 };
18611 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
18612 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
18613 g(&["push", "-q", "origin", "main"]);
18614 g(&["checkout", "-q", "-b", "feat/own"]);
18615 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
18616 g(&["add", "-A"]);
18617 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
18618 let reply = tmp.path().join("reply.txt");
18619 std::fs::write(
18620 &reply,
18621 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
18622 )
18623 .unwrap();
18624 let mut runner = runner_at(RunStatus::Gating);
18625 let mut state = review_state(&["feat(own): add the own file"]);
18626 state.repo = repo.clone();
18627 state.base_branch = "main".to_owned();
18628 state.candidates[0].branch = "feat/own".to_owned();
18629 state.candidates[0].worktree = repo.clone();
18630 let mut author = spec("alpha");
18631 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
18632 author
18633 .env
18634 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
18635 state.config.agents = vec![author];
18636 runner.state = state;
18637 SummaryFixture {
18638 _tmp: tmp,
18639 repo,
18640 reply,
18641 runner,
18642 }
18643 }
18644
18645 async fn posted(f: &mut SummaryFixture) -> PrMessage {
18646 let winner = f.runner.state.candidates[0].clone();
18647 f.runner
18648 .guarded_pr_message(&winner, None, true)
18649 .await
18650 .unwrap()
18651 .expect("an English description never parks")
18652 }
18653
18654 fn clean_round_with_minor_findings() -> ReviewRound {
18655 let mut round = passing_round(2, None);
18656 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
18657 round.reviews = vec![ReviewRecord {
18658 attempts: 0,
18659 reviewer: 1,
18660 agent: "alpha".into(),
18661 summary: "内部レビュー".into(),
18662 findings: vec![
18663 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
18664 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
18665 ],
18666 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
18667 failed: None,
18668 duration_ms: 0,
18669 }];
18670 round
18671 }
18672
18673 #[tokio::test]
18674 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
18675 for review_only in [false, true] {
18676 let mut f = summary_fixture();
18677 if !review_only {
18678 f.runner.state.candidates[0].agent = "alpha".into();
18679 }
18680 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
18681 assert!(f.runner.state.reviews[0].clean);
18682 assert!(f.runner.state.reviews[0].fix.is_none());
18683 assert!(f.runner.state.open_findings().is_empty());
18684 assert_eq!(
18685 crate::pr_outcome::required_finding_ids(&f.runner.state),
18686 ["R2-1-1", "R2-1-2"]
18687 );
18688 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18689 let evidence = crate::pr_outcome::summary_evidence(
18690 &f.runner.state,
18691 crate::pr_outcome::FinalHead::At(&head),
18692 );
18693 assert!(
18694 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
18695 );
18696 let omitted = posted(&mut f).await;
18698 assert!(omitted.body.contains("No description"));
18699 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
18700 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
18701 assert!(f.runner.state.change_summary.is_none());
18702 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
18703 let described = posted(&mut f).await;
18704 assert!(described.body.contains("improve documentation for callers"));
18705 assert!(described.body.contains("remaining edge case"));
18706 assert!(described.body.contains("## Open review findings"));
18707 assert!(!described.body.contains("未修正"));
18708 }
18709 }
18710
18711 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
18712 let mut f = summary_fixture();
18713 if !review_only {
18714 f.runner.state.candidates[0].agent = "alpha".into();
18715 }
18716 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18717 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
18718 fixed
18719 .fix
18720 .as_mut()
18721 .unwrap()
18722 .rejected
18723 .push(crate::verdict::Rejection {
18724 id: "R1-1-1".into(),
18725 why: "呼び出されないコードのため辞退".into(),
18726 });
18727 let mut clean = clean_round_with_minor_findings();
18728 clean.verified_head = Some(head.clone());
18729 f.runner.state.reviews = vec![fixed, clean];
18730 ok_gate(&mut f.runner.state, Some(&head));
18731 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
18732 f.runner.state.config.agents.push(scripted("beta", reply));
18733 f.runner.state.config.roles.language_judge =
18734 Some(crate::config::AgentChoice::One("beta".into()));
18735 f
18736 }
18737
18738 #[tokio::test]
18739 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
18740 for review_only in [false, true] {
18741 let mut f = rewrite_fixture_with_findings(review_only,
18742 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
18743 let m = posted(&mut f).await;
18744 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18745 let expected =
18746 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
18747 assert!(m.body.contains(&expected));
18748 assert!(m.body.contains("## Review rounds"));
18749 assert!(
18750 m.body
18751 .contains("Gate (`verify.gate`): passed on the final commit")
18752 );
18753 assert!(
18754 m.body
18755 .contains("Tests (`verify.e2e`): passed on the final commit")
18756 );
18757 let c = f.runner.state.change_summary.as_ref().unwrap();
18758 assert!(
18759 m.body
18760 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
18761 );
18762 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18763 assert!(crate::pr_outcome::covers_required_findings(
18764 &f.runner.state,
18765 &m.body
18766 ));
18767 assert!(m.body.contains("documentation remains incomplete"));
18768 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
18769 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
18770 assert!(f.runner.state.github_text.is_none());
18771 }
18772 }
18773
18774 #[tokio::test]
18775 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
18776 for review_only in [false, true] {
18777 for reply in [
18778 "TITLE: fix: public description\nA short English description.",
18779 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
18780 ] {
18781 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
18782 let winner = f.runner.state.candidates[0].clone();
18783 let posted = f
18784 .runner
18785 .guarded_pr_message(&winner, None, true)
18786 .await
18787 .unwrap();
18788 assert!(posted.is_none());
18789 assert!(f.runner.state.parked);
18790 let withheld = f.runner.state.github_text.as_ref().unwrap();
18791 assert!(
18792 withheld.rewrite.is_none(),
18793 "an incomplete rewrite must not be selectable"
18794 );
18795 let q = ask::Questions::open()
18796 .get(withheld.question.as_ref().unwrap())
18797 .unwrap();
18798 assert!(
18799 !q.choices
18800 .contains(&crate::github_text::USE_REWRITE.to_owned())
18801 );
18802 assert!(
18803 q.detail.contains("R1-1-1")
18804 && q.detail.contains("R2-1-1")
18805 && q.detail.contains("R2-1-2")
18806 );
18807 }
18808 }
18809 }
18810
18811 #[tokio::test]
18812 async fn japanese_reports_stay_local_in_both_public_run_modes() {
18813 for review_only in [false, true] {
18814 let mut f = summary_fixture();
18815 if !review_only {
18816 f.runner.state.candidates[0].agent = "alpha".into();
18817 }
18818 f.runner.state.instruction = "内部タスクの説明".into();
18819 f.runner.state.candidates[0].summary = "内部の候補報告".into();
18820 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18821 f.runner.state.reviews = vec![
18822 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18823 round_with_notes(2, Some("二回目の内部修正報告です。")),
18824 passing_round(3, Some(&head)),
18825 ];
18826 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18827 ok_gate(&mut f.runner.state, Some(&head));
18828 let captured = f.repo.join("public-prompt.txt");
18829 let author = &mut f.runner.state.config.agents[0];
18830 author.command = vec![
18831 "sh".into(),
18832 "-c".into(),
18833 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18834 ];
18835 author.env.insert(
18836 "PROMPT_FILE".into(),
18837 captured.to_string_lossy().into_owned(),
18838 );
18839 let m = posted(&mut f).await;
18840 assert!(m.body.contains("Adds `own.txt`"));
18841 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18842 assert!(m.body.contains("R1-1-1"));
18843 assert!(
18844 m.body
18845 .contains("Gate (`verify.gate`): passed on the final commit")
18846 );
18847 assert!(
18848 m.body
18849 .contains("Tests (`verify.e2e`): passed on the final commit")
18850 );
18851 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18852 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18853 let prompt = std::fs::read_to_string(captured).unwrap();
18854 assert!(prompt.contains(&format!(
18855 "git diff {} {head}",
18856 f.runner.state.change_summary.as_ref().unwrap().base
18857 )));
18858 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18859 assert_eq!(
18860 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18861 "二回目の内部修正報告です。"
18862 );
18863 }
18864 }
18865
18866 #[tokio::test]
18867 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18868 for review_only in [false, true] {
18869 let mut f = summary_fixture();
18870 if !review_only {
18871 f.runner.state.candidates[0].agent = "alpha".into();
18872 }
18873 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18874 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18875 std::fs::write(&f.reply, "no usable description").unwrap();
18876 let failed = posted(&mut f).await;
18877 assert!(
18878 failed
18879 .body
18880 .contains("No description of what this change does could be generated")
18881 );
18882 assert!(failed.body.is_ascii());
18883 assert!(f.runner.state.github_text.is_none());
18884 assert!(f.runner.state.change_summary.is_none());
18885 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18886 let evidence = crate::pr_outcome::summary_evidence(
18887 &f.runner.state,
18888 crate::pr_outcome::FinalHead::At(&head),
18889 );
18890 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18891 assert!(
18892 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
18893 .exists()
18894 );
18895 f.runner.state =
18896 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
18897 std::fs::write(
18898 &f.reply,
18899 "TITLE: feat(own): add the own file\nAdds the committed own file.",
18900 )
18901 .unwrap();
18902 let retried = posted(&mut f).await;
18903 assert!(retried.body.contains("Adds the committed own file."));
18904 assert_eq!(
18905 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
18906 "日本語の修正報告"
18907 );
18908 }
18909 }
18910
18911 #[tokio::test]
18912 async fn same_head_verification_and_findings_invalidate_public_prose() {
18913 let mut f = summary_fixture();
18914 posted(&mut f).await;
18915 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18916 let original = f
18917 .runner
18918 .state
18919 .change_summary
18920 .as_ref()
18921 .unwrap()
18922 .evidence
18923 .clone();
18924 ok_gate(&mut f.runner.state, Some(&head));
18925 std::fs::write(
18926 &f.reply,
18927 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
18928 )
18929 .unwrap();
18930 let verified = posted(&mut f).await;
18931 assert!(verified.body.contains("now verified"));
18932 assert_ne!(
18933 f.runner.state.change_summary.as_ref().unwrap().evidence,
18934 original
18935 );
18936 let mut round = round_with_notes(1, Some("内部報告"));
18937 round.reviews.push(ReviewRecord {
18938 attempts: 0,
18939 reviewer: 1,
18940 agent: "alpha".into(),
18941 summary: "内部レビュー".into(),
18942 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
18943 vote: None,
18944 failed: None,
18945 duration_ms: 0,
18946 });
18947 round
18948 .fix
18949 .as_mut()
18950 .unwrap()
18951 .rejected
18952 .push(crate::verdict::Rejection {
18953 id: "R1-1-1".into(),
18954 why: "呼び出し元から到達できない".into(),
18955 });
18956 f.runner.state.reviews = vec![round];
18957 let omitted = posted(&mut f).await;
18959 assert!(omitted.body.contains("No description"));
18960 assert!(omitted.body.contains("R1-1-1"));
18961 assert!(!omitted.body.contains("到達"));
18962 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
18963 let described = posted(&mut f).await;
18964 assert!(described.body.contains("unreachable from callers"));
18965 assert!(described.body.contains("R1-1-1"));
18966 assert!(!described.body.contains("到達"));
18967 let evidence = f
18968 .runner
18969 .state
18970 .change_summary
18971 .as_ref()
18972 .unwrap()
18973 .evidence
18974 .clone();
18975 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
18976 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
18977 assert!(posted(&mut f).await.body.contains("rationale updated"));
18978 assert_ne!(
18979 f.runner.state.change_summary.as_ref().unwrap().evidence,
18980 evidence
18981 );
18982 f.runner
18984 .state
18985 .change_summary
18986 .as_mut()
18987 .unwrap()
18988 .evidence
18989 .clear();
18990 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
18991 assert!(posted(&mut f).await.body.contains("current specification"));
18992 }
18993
18994 #[tokio::test]
18995 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
18996 let mut f = summary_fixture();
18997 let m = posted(&mut f).await;
18998 assert_eq!(m.title, "feat(own): add the own file");
18999 assert!(
19000 m.body
19001 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19002 "{}",
19003 m.body
19004 );
19005 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19006 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19007 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19008 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19009 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19010 let kept = f.runner.state.change_summary.as_ref().unwrap();
19011 assert_eq!(
19012 (kept.head.as_str(), kept.base.as_str()),
19013 (head.as_str(), base.as_str())
19014 );
19015 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19016 }
19017
19018 #[tokio::test]
19019 async fn review_only_summary_receives_the_configured_github_language() {
19020 let mut f = summary_fixture();
19021 f.runner.state.config.graph.github_language = "ja".into();
19022 let captured = f.repo.join("summary-prompt.txt");
19023 let author = &mut f.runner.state.config.agents[0];
19024 author.command = vec![
19025 "sh".into(),
19026 "-c".into(),
19027 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19028 ];
19029 author.env.insert(
19030 "PROMPT_FILE".into(),
19031 captured.to_string_lossy().into_owned(),
19032 );
19033 std::fs::write(
19034 &f.reply,
19035 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19036 )
19037 .unwrap();
19038 let m = posted(&mut f).await;
19039 let prompt = std::fs::read_to_string(captured).unwrap();
19040 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19041 assert!(!prompt.contains("in English"));
19042 assert_eq!(m.title, "feat(own): ファイルを追加");
19043 assert!(m.body.contains("own.txt を追加します。"));
19044 }
19045
19046 #[tokio::test]
19047 async fn review_fixes_come_after_the_whole_change_summary() {
19048 let mut f = summary_fixture();
19049 f.runner.state.reviews = vec![
19050 round_with_notes(1, Some("handled the PATH case")),
19051 round_with_notes(2, None),
19052 ];
19053 let m = posted(&mut f).await;
19054 let summary = m.body.find("## Summary").expect("summary");
19055 let fixes = m.body.find("## Review rounds").expect("review rounds");
19056 assert!(summary < fixes, "{}", m.body);
19057 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19058 assert!(m.body.contains("## Final verification"), "{}", m.body);
19059 }
19060
19061 #[tokio::test]
19062 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19063 let mut f = summary_fixture();
19064 let first = posted(&mut f).await;
19065 let first_head = f.runner.state.change_summary.clone().unwrap().head;
19066 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19068 let again = posted(&mut f).await;
19069 assert_eq!((again.title, again.body), (first.title, first.body));
19070
19071 let g = |args: &[&str]| {
19072 let out = std::process::Command::new("git")
19073 .args(args)
19074 .current_dir(&f.repo)
19075 .quiet()
19076 .output()
19077 .expect("spawn git");
19078 assert!(out.status.success(), "git {args:?}");
19079 };
19080 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19081 g(&["add", "-A"]);
19082 g(&["commit", "-q", "-m", "fix(own): handle more"]);
19083 let second = posted(&mut f).await;
19084 let kept = f.runner.state.change_summary.clone().unwrap();
19085 assert_ne!(kept.head, first_head);
19086 assert!(second.body.contains("Not read yet."), "{}", second.body);
19087 assert!(!second.body.contains("so the first summary is read"));
19088 assert_eq!(second.title, "feat(own): other");
19089 }
19090
19091 #[tokio::test]
19092 async fn an_unavailable_summary_is_stated_and_never_invented() {
19093 let mut f = summary_fixture();
19094 std::fs::write(&f.reply, "no title line here").unwrap();
19095 let m = posted(&mut f).await;
19096 assert!(
19097 m.body
19098 .contains("No description of what this change does could be generated"),
19099 "{}",
19100 m.body
19101 );
19102 assert!(m.body.contains("internal reports remain in local records"));
19103 assert!(f.runner.state.change_summary.is_none());
19104 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19105 let winner = f.runner.state.candidates[0].clone();
19107 let preview = f
19108 .runner
19109 .guarded_pr_message(&winner, None, false)
19110 .await
19111 .unwrap()
19112 .unwrap();
19113 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19114 }
19115
19116 #[test]
19117 fn pr_message_review_single_commit_uses_its_subject() {
19118 let state = review_state(&["feat(nats): per-role user"]);
19119 let m = pr_message(&state, 'A');
19120 assert_eq!(m.title, "feat(nats): per-role user");
19121 assert!(!m.body.contains("Review the work already"), "{}", m.body);
19122 assert!(m.body.contains("## Commits under review"), "{}", m.body);
19123 }
19124
19125 #[test]
19126 fn pr_message_review_multi_commit_takes_the_oldest() {
19127 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19128 let m = pr_message(&state, 'A');
19129 assert_eq!(m.title, "feat: the change");
19130 for s in ["feat: the change", "fix: typo", "fix: again"] {
19131 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19132 }
19133 }
19134
19135 #[test]
19136 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19137 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19138 let mut state = review_state(&[first, "fix: later fixup"]);
19139 state.candidates[0].branch = "機能/ログイン".to_owned();
19140 let m = pr_message(&state, 'A');
19141 assert!(
19142 m.title.starts_with("chore: land candidate A of run"),
19143 "{}",
19144 m.title
19145 );
19146 }
19147 }
19148
19149 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19150 BranchFacts {
19151 commits: commits
19152 .iter()
19153 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19154 .collect(),
19155 stat: stat.to_owned(),
19156 }
19157 }
19158
19159 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19160 let mut r = review_round(true, 0, 1, 1, true, true);
19161 r.round = round;
19162 r.fix = notes.map(|n| FixRecord {
19163 agent: "fixer".to_owned(),
19164 addressed: Vec::new(),
19165 rejected: Vec::new(),
19166 notes: n.to_owned(),
19167 committed: true,
19168 failed: None,
19169 duration_ms: 0,
19170 continuation: None,
19171 });
19172 r
19173 }
19174
19175 #[test]
19176 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19177 let state = review_state(&["ignored"]);
19178 let f = facts(
19179 &[
19180 (
19181 "fix(login): resolve PATH on macOS",
19182 "Login shells skip rc files.",
19183 ),
19184 ("fix: address review", ""),
19185 ],
19186 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19187 );
19188 let m = pr_message_with(&state, 'A', Some(&f));
19189 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19190 assert!(
19191 m.body
19192 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
19193 "{}",
19194 m.body
19195 );
19196 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19197 assert!(m.body.contains("## Diff stat"), "{}", m.body);
19198 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19199 for banned in [
19200 "Review the work already",
19201 "no task statement",
19202 "Original task",
19203 ] {
19204 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19205 }
19206 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19207 }
19208
19209 #[test]
19210 fn pr_message_review_truncates_a_huge_first_commit_body() {
19211 let state = review_state(&["ignored"]);
19212 let f = facts(
19213 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19214 "s",
19215 );
19216 let m = pr_message_with(&state, 'A', Some(&f));
19217 assert!(m.body.len() < 30_000, "{}", m.body.len());
19218 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19219 assert!(
19220 m.body.contains("- ... 1 more commit(s)"),
19221 "{}",
19222 m.body.len()
19223 );
19224 assert!(m.body.ends_with("magi:candidate-a\n"));
19225 }
19226
19227 #[test]
19228 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19229 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19230 assert_eq!(m.title, "feat: x");
19231 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19232 assert!(!m.body.contains("Diff stat"), "{}", m.body);
19233 assert!(!m.body.contains("no task statement"), "{}", m.body);
19234 }
19235
19236 #[test]
19237 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19238 let mut state = review_state(&["日本語の件名"]);
19239 state.candidates[0].branch = "fix/macos-login-path".to_owned();
19240 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19241 state.candidates[0].branch = "機能/ログイン".to_owned();
19242 assert!(
19243 pr_message(&state, 'A')
19244 .title
19245 .starts_with("chore: land candidate A")
19246 );
19247 }
19248
19249 #[test]
19250 fn pr_message_review_fixes_survive_a_clean_final_round() {
19251 let mut state = review_state(&["feat: x"]);
19252 state.reviews = vec![
19253 round_with_notes(1, Some("handled the PATH case")),
19254 round_with_notes(2, None),
19255 ];
19256 let body = pr_message(&state, 'A').body;
19257 assert!(
19258 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19259 "{body}"
19260 );
19261 assert!(body.contains("### Round 1"), "{body}");
19262
19263 state.reviews = vec![
19264 round_with_notes(1, Some("first fix")),
19265 round_with_notes(2, Some("")),
19266 round_with_notes(3, Some("second fix")),
19267 round_with_notes(4, None),
19268 ];
19269 let body = pr_message(&state, 'A').body;
19270 assert!(
19271 body.contains("### Round 1") && !body.contains("first fix"),
19272 "{body}"
19273 );
19274 assert!(
19275 body.contains("### Round 3") && !body.contains("second fix"),
19276 "{body}"
19277 );
19278 assert!(body.contains("### Round 2"), "{body}");
19279 }
19280
19281 const FINAL: &str = "abcdef1234567890";
19282
19283 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
19284 let mut r = round_with_notes(round, None);
19285 r.verified_head = verified.map(str::to_owned);
19286 r
19287 }
19288
19289 fn ok_gate(state: &mut RunState, head: Option<&str>) {
19290 state.gate = vec![CommandOutcome {
19291 command: "gate".to_owned(),
19292 code: Some(0),
19293 output_tail: String::new(),
19294 duration_ms: 0,
19295 resource_blocked: false,
19296 }];
19297 state.gate_ran = true;
19298 state.gate_head = head.map(str::to_owned);
19299 }
19300
19301 fn body_at(state: &RunState) -> String {
19302 pr_message_raw(
19303 state,
19304 'A',
19305 None,
19306 crate::pr_outcome::FinalHead::At(FINAL),
19307 ChangeNote::NotAsked,
19308 )
19309 .body
19310 }
19311
19312 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
19313
19314 #[test]
19315 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
19316 let mut state = review_state(&["feat: x"]);
19317 state.reviews = vec![
19318 round_with_notes(1, Some(SANDBOXED)),
19319 passing_round(2, Some(FINAL)),
19320 ];
19321 ok_gate(&mut state, Some(FINAL));
19322 let body = body_at(&state);
19323 assert!(!body.contains("Handled the PATH case."), "{body}");
19324 assert!(!body.contains("could not commit"), "{body}");
19325 assert!(!body.contains("could not run"), "{body}");
19326 assert!(
19327 body.contains("The fix is committed on the branch"),
19328 "{body}"
19329 );
19330 assert!(!body.contains("sandbox"), "{body}");
19331 assert!(
19332 body.contains("Tests (`verify.e2e`): passed on the final commit"),
19333 "{body}"
19334 );
19335 assert!(
19336 body.contains("Gate (`verify.gate`): passed on the final commit"),
19337 "{body}"
19338 );
19339 assert!(
19341 state.reviews[0]
19342 .fix
19343 .as_ref()
19344 .unwrap()
19345 .notes
19346 .contains("could not commit")
19347 );
19348 }
19349
19350 #[test]
19351 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
19352 let mut state = review_state(&["feat: x"]);
19353 let mut r1 = round_with_notes(1, Some(SANDBOXED));
19354 r1.fix.as_mut().unwrap().committed = false;
19355 let mut r2 = passing_round(2, Some(FINAL));
19356 r2.e2e[0].code = Some(1);
19357 state.reviews = vec![r1, r2];
19358 let body = body_at(&state);
19359 assert!(!body.contains("could not commit"), "{body}");
19360 assert!(!body.contains("could not run"), "{body}");
19361 assert!(body.contains("This round produced no commit"), "{body}");
19362 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
19363 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
19364 assert!(!body.contains("passed"), "{body}");
19365 }
19366
19367 #[test]
19368 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
19369 let mut state = review_state(&["feat: x"]);
19370 state.reviews = vec![
19371 round_with_notes(1, Some(SANDBOXED)),
19372 passing_round(2, Some("0123456789abcdef")),
19373 ];
19374 ok_gate(&mut state, Some("0123456789abcdef"));
19375 let body = body_at(&state);
19376 assert!(!body.contains("HTTP tests could not run"), "{body}");
19377 assert!(!body.contains("could not commit"), "{body}");
19379 assert!(
19380 body.contains("passed on an earlier commit (`0123456`)"),
19381 "{body}"
19382 );
19383 assert!(!body.contains("passed on the final commit"), "{body}");
19384
19385 ok_gate(&mut state, None);
19387 let body = body_at(&state);
19388 assert!(
19389 body.contains("passed on a commit that was not recorded"),
19390 "{body}"
19391 );
19392 }
19393
19394 #[test]
19395 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
19396 let state = state_with_summary(
19397 "add retries to the client",
19398 "TITLE: feat: retries\n\nDid it.",
19399 );
19400 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
19401 assert_eq!(m.title, "feat: retries");
19402 assert!(
19403 m.body.contains("<summary>Original task</summary>"),
19404 "{}",
19405 m.body
19406 );
19407 assert!(!m.body.contains("Commits under review"), "{}", m.body);
19408 assert!(
19409 m.body
19410 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
19411 "{}",
19412 m.body
19413 );
19414 }
19415
19416 #[test]
19417 fn pr_message_review_bounds_a_long_english_subject() {
19418 let long = format!("feat: {}", "word ".repeat(100));
19419 let m = pr_message(&review_state(&[&long]), 'A');
19420 assert!(m.title.starts_with("feat: word"), "{}", m.title);
19421 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19422 }
19423
19424 #[test]
19425 fn pr_message_implementation_run_is_unchanged_by_review_support() {
19426 let state = state_with_summary("add retries\n\ndetails", "- did some things");
19427 let m = pr_message(&state, 'A');
19428 assert_eq!(m.title, "add retries");
19429 assert!(m.body.contains("<summary>Original task</summary>"));
19430 assert!(!m.body.contains("Commits under review"));
19431 assert_eq!(landing_subject_source(&state), state.instruction);
19432 }
19433
19434 #[test]
19435 fn review_run_squash_subject_is_the_change_not_the_prompt() {
19436 let state = review_state(&["feat: the change", "fix: typo"]);
19437 let source = landing_subject_source(&state);
19438 assert_eq!(land::merge_subject("", &source), "feat: the change");
19439 assert_eq!(
19440 land::merge_subject("magi: candidate A (uncommitted work)", &source),
19441 "feat: the change"
19442 );
19443 assert_eq!(
19445 land::merge_subject("feat: renamed by hand", &source),
19446 "feat: renamed by hand"
19447 );
19448 let blank = review_state(&["日本語"]);
19449 assert!(
19450 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
19451 );
19452 }
19453
19454 #[test]
19455 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
19456 let state = review_state(&["feat: the change"]);
19457 let source = landing_subject_source(&state);
19458 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
19459 assert_eq!(
19460 land::merge_subject(landing_title(&state, old), &source),
19461 "feat: the change"
19462 );
19463 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
19464 let task = state_with_summary("add retries", "");
19465 assert_eq!(landing_title(&task, old), old);
19466 }
19467
19468 #[test]
19469 fn pr_message_describes_the_change_not_the_task() {
19470 let state = state_with_summary(
19471 "今回やってほしいこと: results projector を直す",
19472 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
19473 );
19474 let m = pr_message(&state, 'A');
19475 assert_eq!(m.title, "fix(web): batch the runs list reads");
19476 assert!(
19477 m.body.starts_with("## Summary\n\n- reads run.json once"),
19478 "{}",
19479 m.body
19480 );
19481 assert!(!m.body.contains("TITLE:"), "{}", m.body);
19482 let task_at = m.body.find("今回やってほしいこと").unwrap();
19483 let details_at = m.body.find("<details>").unwrap();
19484 assert!(
19485 details_at < task_at,
19486 "the task lives inside <details>: {}",
19487 m.body
19488 );
19489 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
19490 assert!(m.body.contains("magi:candidate-a"));
19491 }
19492
19493 #[test]
19494 fn pr_message_falls_back_to_the_task_without_a_title_line() {
19495 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
19496 let m = pr_message(&state, 'A');
19497 assert_eq!(m.title, "add retries");
19498 assert!(
19499 m.body.contains("## Summary\n\n- did some things"),
19500 "{}",
19501 m.body
19502 );
19503
19504 let none = RunState::new(
19505 PathBuf::from("/repo"),
19506 "main".to_owned(),
19507 "abc1234".to_owned(),
19508 "add retries".to_owned(),
19509 Config::default(),
19510 );
19511 let m = pr_message(&none, 'A');
19512 assert_eq!(m.title, "add retries");
19513 assert!(!m.body.contains("## Summary"), "{}", m.body);
19514 }
19515
19516 #[test]
19517 fn pr_message_refuses_the_candidate_commit_subject() {
19518 for bad in [
19519 "TITLE: magi: candidate A (uncommitted work)",
19520 "TITLE: chore: stuff (uncommitted work)",
19521 "TITLE: ",
19522 ] {
19523 let state = state_with_summary("add retries", bad);
19524 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
19525 }
19526 }
19527
19528 #[test]
19529 fn pr_message_bounds_a_very_long_task_and_title() {
19530 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
19531 let state = state_with_summary(&long, "- nothing");
19532 let m = pr_message(&state, 'A');
19533 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19534 assert!(!m.title.contains('\n'));
19535
19536 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
19537 let m = pr_message(&state, 'A');
19538 assert!(m.title.starts_with("feat: "));
19539 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19540 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
19541 }
19542
19543 fn long_title_of(instruction: &str) -> String {
19544 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
19545 }
19546
19547 #[test]
19548 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
19549 let first = "Make the landing path keep a readable title for long tasks";
19550 let line = format!(
19551 "{first}. {}",
19552 "And then keep going with more words ".repeat(20)
19553 );
19554 let t = long_title_of(&line);
19555 assert_eq!(t, first);
19556 assert!(!t.starts_with("chore: land"));
19557 }
19558
19559 #[test]
19560 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
19561 let line = "word ".repeat(200);
19562 let t = long_title_of(&line);
19563 assert!(t.ends_with("word..."), "{t}");
19564 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
19565 }
19566
19567 #[test]
19568 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
19569 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
19570 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
19571 }
19572 }
19573
19574 #[test]
19575 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
19576 let mut state = state_with_summary("再試行を追加する", "- nothing");
19577 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
19578 state.config.graph.github_language = "ja".into();
19579 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
19580 }
19581
19582 #[test]
19583 fn pr_message_title_limit_is_exact() {
19584 let at = "a".repeat(PR_TITLE_MAX);
19585 assert_eq!(long_title_of(&at), at);
19586 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
19587 assert!(over.ends_with("..."), "{over}");
19588 assert_eq!(over.chars().count(), PR_TITLE_MAX);
19589 }
19590
19591 #[test]
19592 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
19593 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
19594 let t = long_title_of(&line);
19595 assert!(t.ends_with("..."), "{t}");
19596 assert!(t.is_ascii(), "{t}");
19597 }
19598
19599 #[test]
19600 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
19601 let line = format!(
19602 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
19603 "plus more filler words ".repeat(20)
19604 );
19605 let t = long_title_of(&line);
19606 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
19607 }
19608
19609 #[test]
19610 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
19611 let line = format!("\n\n# {}", "title words ".repeat(40));
19612 let state = state_with_summary(&line, "- nothing");
19613 let m = pr_message(&state, 'A');
19614 assert!(m.title.starts_with("title words"), "{}", m.title);
19615 assert_eq!(
19616 land::merge_subject(&m.title, &landing_subject_source(&state)),
19617 m.title
19618 );
19619 assert_eq!(
19621 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
19622 "feat: renamed by hand"
19623 );
19624 }
19625
19626 #[tokio::test]
19627 async fn github_text_rewrite_is_bounded_and_falls_back() {
19628 crate::run::pin_test_home();
19629 for (reply, accepted) in [
19630 (
19631 "TITLE: fix: retries\nAdd retries for failed requests.",
19632 true,
19633 ),
19634 (
19635 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
19636 false,
19637 ),
19638 (
19639 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
19640 true,
19641 ),
19642 (
19643 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
19644 true,
19645 ),
19646 (
19647 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
19648 true,
19649 ),
19650 (
19651 "TITLE: fix: retries\n日本語の説明です。 token=secret",
19652 false,
19653 ),
19654 ("TITLE: fix: retries\n", false),
19655 ] {
19656 let dir = tempfile::tempdir().unwrap();
19657 let mut runner = runner_at(RunStatus::Gating);
19658 runner.state = state_with_summary(
19659 "add retries",
19660 "TITLE: fix: retries\n日本語の説明を書きます。",
19661 );
19662 runner.state.repo = dir.path().to_owned();
19663 runner.state.candidates[0].worktree = dir.path().to_owned();
19664 let mut author = spec("alpha");
19665 author.command = vec![
19666 "sh".into(),
19667 "-c".into(),
19668 "printf '%s' \"$REWRITE_REPLY\"".into(),
19669 ];
19670 author.env.insert("REWRITE_REPLY".into(), reply.into());
19671 runner.state.config.agents = vec![author];
19672 seed_public_summary(&mut runner);
19673 let winner = runner.state.candidates[0].clone();
19674 let first = runner
19675 .guarded_pr_message(&winner, None, true)
19676 .await
19677 .unwrap();
19678 let message = if accepted {
19679 first.expect("a passing rewrite never asks")
19680 } else {
19681 assert!(first.is_none());
19683 assert!(runner.state.parked);
19684 let store = ask::Questions::open();
19685 let asked: Vec<_> = store
19686 .list()
19687 .into_iter()
19688 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19689 .collect();
19690 assert_eq!(asked.len(), 1);
19691 assert!(!asked[0].detail.contains("secret"));
19692 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
19693 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
19694 assert_eq!(
19695 asked[0]
19696 .choices
19697 .contains(&crate::github_text::USE_REWRITE.to_owned()),
19698 rewrite.is_some()
19699 );
19700 store
19703 .update(&asked[0].id, |q| {
19704 q.abandon("test");
19705 Ok(())
19706 })
19707 .unwrap();
19708 let again = runner
19709 .guarded_pr_message(&winner, None, true)
19710 .await
19711 .unwrap()
19712 .expect("an abandoned question falls back");
19713 let n = store
19714 .list()
19715 .into_iter()
19716 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19717 .count();
19718 assert_eq!(n, 1, "asked once per rejected text");
19719 again
19720 };
19721 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
19722 assert!(!message.body.contains("secret"));
19723 assert!(!message.body.contains("prefix"));
19724 assert!(!message.title.contains("secret"));
19725 assert_eq!(
19726 message.body.contains("Add retries for failed requests."),
19727 accepted
19728 );
19729 assert_eq!(
19730 runner.state.seats["github-text-rewrite"].turns, 1,
19731 "only one rewrite invocation"
19732 );
19733 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
19734 if accepted {
19735 assert!(runner.state.github_text.is_none());
19736 assert!(!runner.state.parked);
19737 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
19738 assert!(message.body.contains("## Final verification"));
19739 assert!(message.body.contains("Evidence: final diff"));
19740 assert!(
19741 message
19742 .body
19743 .contains(&format!("magi:run/{}", runner.state.id))
19744 );
19745 }
19746 if !accepted {
19747 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19748 assert!(
19749 message
19750 .body
19751 .contains(&format!("magi:run/{}", runner.state.id))
19752 );
19753 }
19754 }
19755 }
19756
19757 fn scripted(id: &str, reply: &str) -> AgentSpec {
19758 let mut a = spec(id);
19759 a.command = vec![
19760 "sh".into(),
19761 "-c".into(),
19762 "printf '%s' \"$REWRITE_REPLY\"".into(),
19763 ];
19764 a.env.insert("REWRITE_REPLY".into(), reply.into());
19765 a
19766 }
19767
19768 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
19769 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
19770
19771 fn seed_public_summary(runner: &mut Runner) {
19773 let repo = &runner.state.repo;
19774 let g = |args: &[&str]| {
19775 let out = std::process::Command::new("git")
19776 .args(args)
19777 .current_dir(repo)
19778 .quiet()
19779 .output()
19780 .unwrap();
19781 assert!(out.status.success(), "git {args:?}");
19782 String::from_utf8_lossy(&out.stdout).trim().to_owned()
19783 };
19784 g(&["init", "-q", "-b", "main"]);
19785 g(&[
19786 "-c",
19787 "user.name=Test",
19788 "-c",
19789 "user.email=test@example.invalid",
19790 "commit",
19791 "-q",
19792 "--allow-empty",
19793 "-m",
19794 "feat: fixture",
19795 ]);
19796 let head = g(&["rev-parse", "HEAD"]);
19797 g(&["update-ref", "refs/remotes/origin/main", &head]);
19798 let evidence = crate::pr_outcome::summary_evidence(
19799 &runner.state,
19800 crate::pr_outcome::FinalHead::At(&head),
19801 );
19802 let generated = &runner.state.candidates[0].summary;
19803 runner.state.change_summary = Some(crate::run::ChangeSummary {
19804 base: head.clone(),
19805 head,
19806 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
19807 title: summary_title(generated).unwrap(),
19808 body: summary_without_title(generated),
19809 });
19810 }
19811
19812 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19813 let mut runner = runner_at(RunStatus::Gating);
19814 runner.state = state_with_summary("add retries", JA_SUMMARY);
19815 if review_only {
19816 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19817 }
19818 runner.state.repo = dir.to_owned();
19819 runner.state.candidates[0].worktree = dir.to_owned();
19820 runner.state.config.agents = agents;
19821 seed_public_summary(&mut runner);
19822 runner
19823 }
19824
19825 fn gate_questions(runner: &Runner) -> usize {
19826 ask::Questions::open()
19827 .list()
19828 .into_iter()
19829 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19830 .count()
19831 }
19832
19833 #[tokio::test]
19834 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19835 crate::run::pin_test_home();
19836 let dir = tempfile::tempdir().unwrap();
19837 let mut runner = rewrite_runner(
19838 dir.path(),
19839 true,
19840 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19841 );
19842 runner.state.config.roles.language_judge =
19843 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19844 let winner = runner.state.candidates[0].clone();
19845 let msg = runner
19846 .guarded_pr_message(&winner, None, true)
19847 .await
19848 .unwrap()
19849 .expect("an English rewrite posts without asking");
19850 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19851 assert!(msg.body.contains("<details>kept</details>"));
19852 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19853 assert_eq!(gate_questions(&runner), 0);
19854 assert!(runner.state.github_text.is_none());
19855 assert!(!runner.state.seats.contains_key("impl-A"));
19856 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19857 }
19858
19859 #[tokio::test]
19860 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19861 crate::run::pin_test_home();
19862 let dir = tempfile::tempdir().unwrap();
19863 let mut runner = rewrite_runner(
19864 dir.path(),
19865 false,
19866 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19867 );
19868 runner.state.config.roles.language_judge =
19869 Some(crate::config::AgentChoice::One("beta".into()));
19870 runner.state.seats.insert(
19871 "impl-A".into(),
19872 SeatState::new("impl-A", "alpha", runner.state.seed),
19873 );
19874 let winner = runner.state.candidates[0].clone();
19875 let msg = runner
19876 .guarded_pr_message(&winner, None, true)
19877 .await
19878 .unwrap()
19879 .expect("rewritten");
19880 assert!(msg.body.contains("<details>kept</details>"));
19881 assert_eq!(runner.state.seats["impl-A"].turns, 0);
19882 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19883 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19884 }
19885
19886 #[tokio::test]
19887 async fn rewrite_reuses_a_resumable_implementer_session() {
19888 crate::run::pin_test_home();
19889 let dir = tempfile::tempdir().unwrap();
19890 let mut runner = rewrite_runner(
19891 dir.path(),
19892 false,
19893 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
19894 );
19895 runner.state.config.roles.language_judge =
19896 Some(crate::config::AgentChoice::One("beta".into()));
19897 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19898 seat.turns = 1;
19899 runner.state.seats.insert("impl-A".into(), seat);
19900 let winner = runner.state.candidates[0].clone();
19901 runner
19902 .guarded_pr_message(&winner, None, true)
19903 .await
19904 .unwrap()
19905 .expect("rewritten");
19906 assert_eq!(runner.state.seats["impl-A"].turns, 2);
19907 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
19908 }
19909
19910 #[tokio::test]
19911 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
19912 crate::run::pin_test_home();
19915 let dir = tempfile::tempdir().unwrap();
19916 let mut exhausted = spec("alpha");
19917 exhausted.command = vec!["false".into()];
19918 let mut runner = rewrite_runner(
19919 dir.path(),
19920 false,
19921 vec![exhausted, scripted("beta", EN_REWRITE)],
19922 );
19923 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19925 "alpha".into(),
19926 "beta".into(),
19927 ]));
19928 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19929 seat.turns = 1;
19930 runner.state.seats.insert("impl-A".into(), seat);
19931 let winner = runner.state.candidates[0].clone();
19932 let msg = runner
19933 .guarded_pr_message(&winner, None, true)
19934 .await
19935 .unwrap()
19936 .expect("the role rewrites after the implementer fails");
19937 assert!(msg.body.contains("`src/net.rs`"));
19938 assert_eq!(
19939 runner.state.seats["impl-A"].turns, 2,
19940 "the implementer was tried once"
19941 );
19942 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19943 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19944 }
19945
19946 #[tokio::test]
19947 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
19948 crate::run::pin_test_home();
19949 let dir = tempfile::tempdir().unwrap();
19950 let mut broken = spec("bad");
19951 broken.command = vec!["false".into()];
19952 let mut runner = rewrite_runner(
19953 dir.path(),
19954 true,
19955 vec![
19956 broken,
19957 scripted("noformat", "Add retries."),
19958 scripted("good", EN_REWRITE),
19959 ],
19960 );
19961 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19962 "bad".into(),
19963 "noformat".into(),
19964 "good".into(),
19965 ]));
19966 let winner = runner.state.candidates[0].clone();
19967 let msg = runner
19968 .guarded_pr_message(&winner, None, true)
19969 .await
19970 .unwrap()
19971 .expect("the third agent rewrites");
19972 assert!(msg.body.contains("`src/net.rs`"));
19973 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
19974 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
19975 }
19976
19977 #[tokio::test]
19978 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
19979 crate::run::pin_test_home();
19980 for (agents, role, saved) in [
19982 (
19983 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
19984 Some("beta"),
19985 true,
19986 ),
19987 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
19988 (Vec::new(), None, false),
19989 ] {
19990 let dir = tempfile::tempdir().unwrap();
19991 let mut runner = rewrite_runner(dir.path(), true, agents);
19992 runner.state.config.roles.language_judge =
19993 role.map(|r| crate::config::AgentChoice::One(r.into()));
19994 let winner = runner.state.candidates[0].clone();
19995 let first = runner
19996 .guarded_pr_message(&winner, None, true)
19997 .await
19998 .unwrap();
19999 assert!(first.is_none(), "withheld text parks the run");
20000 assert!(runner.state.parked);
20001 assert_eq!(gate_questions(&runner), 1);
20002 let g = runner.state.github_text.as_ref().unwrap();
20003 assert_eq!(g.rewrite.is_some(), saved);
20004 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20005 }
20006 }
20007
20008 #[tokio::test]
20009 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20010 crate::run::pin_test_home();
20011 let dir = tempfile::tempdir().unwrap();
20012 let mut runner = runner_at(RunStatus::Gating);
20013 runner.state =
20014 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20015 runner.state.repo = dir.path().to_owned();
20016 runner.state.candidates[0].worktree = dir.path().to_owned();
20017 let mut author = spec("alpha");
20018 author.command = vec![
20019 "sh".into(),
20020 "-c".into(),
20021 "printf '%s' \"$REWRITE_REPLY\"".into(),
20022 ];
20023 author.env.insert(
20024 "REWRITE_REPLY".into(),
20025 format!(
20026 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20027 "English sentence. ".repeat(600)
20028 ),
20029 );
20030 runner.state.config.agents = vec![author];
20031 seed_public_summary(&mut runner);
20032 let winner = runner.state.candidates[0].clone();
20033 assert!(
20034 runner
20035 .guarded_pr_message(&winner, None, true)
20036 .await
20037 .unwrap()
20038 .is_none()
20039 );
20040 let record = runner.state.github_text.clone().unwrap();
20041 let rewrite = record.rewrite.clone().unwrap();
20042 assert_eq!(rewrite.failed, ["body-language"]);
20043 assert!(
20044 rewrite
20045 .body
20046 .ends_with(&format!("magi:run/{}", runner.state.id))
20047 );
20048 let store = ask::Questions::open();
20049 let q = store.get(record.question.as_ref().unwrap()).unwrap();
20050 assert!(
20051 q.choices
20052 .contains(&crate::github_text::USE_REWRITE.to_owned())
20053 );
20054 for label in [
20055 "ORIGINAL",
20056 "FALLBACK",
20057 "REWRITE",
20058 "Rules still failed: body-language",
20059 "truncated",
20060 ] {
20061 assert!(q.detail.contains(label), "{label}");
20062 }
20063 let artifact = std::fs::read_to_string(crate::run::artifact_path(
20064 &runner.state,
20065 &crate::github_text::artifact_name(&record.fingerprint),
20066 ))
20067 .unwrap();
20068 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20069 runner.state =
20071 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20072 store
20073 .update(&q.id, |q| {
20074 q.say("まだ日本語")?;
20075 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20076 })
20077 .unwrap();
20078 assert!(
20079 runner
20080 .guarded_pr_message(&winner, None, true)
20081 .await
20082 .unwrap()
20083 .is_none()
20084 );
20085 let retry = runner.state.github_text.as_ref().unwrap();
20086 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20087 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20088 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20089 assert!(
20090 q.posting_comparison
20091 .as_ref()
20092 .unwrap()
20093 .matches_detail(&q.detail)
20094 );
20095 store
20096 .update(&q.id, |q| {
20097 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20098 })
20099 .unwrap();
20100 let message = runner
20101 .guarded_pr_message(&winner, None, true)
20102 .await
20103 .unwrap()
20104 .unwrap();
20105 assert_eq!(message.title, rewrite.title);
20106 assert_eq!(message.body, rewrite.body);
20107 assert_eq!(
20108 runner.state.github_text.as_ref().unwrap().choice,
20109 Some(GithubTextChoice::Rewrite)
20110 );
20111 let again = runner
20112 .guarded_pr_message(&winner, None, true)
20113 .await
20114 .unwrap()
20115 .unwrap();
20116 assert_eq!(again.title, message.title);
20117 assert_eq!(again.body, message.body);
20118 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20119 assert!(
20120 !runner
20121 .state
20122 .events
20123 .iter()
20124 .any(|e| e.message.contains(&rewrite.title)
20125 || e.message.contains("English sentence."))
20126 );
20127 }
20128
20129 #[tokio::test]
20130 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20131 crate::run::pin_test_home();
20132 let dir = tempfile::tempdir().unwrap();
20133 let mut runner = runner_at(RunStatus::Gating);
20134 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20135 runner.state.repo = dir.path().to_owned();
20136 runner.state.candidates[0].worktree = dir.path().to_owned();
20137 seed_public_summary(&mut runner);
20138 let winner = runner.state.candidates[0].clone();
20139 assert!(
20140 runner
20141 .guarded_pr_message(&winner, None, true)
20142 .await
20143 .unwrap()
20144 .is_none()
20145 );
20146 let record = runner.state.github_text.as_ref().unwrap();
20147 assert!(record.rewrite.is_none());
20148 let mut old = serde_json::to_value(record).unwrap();
20150 old.as_object_mut().unwrap().remove("rewrite");
20151 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20152 assert!(old.rewrite.is_none());
20153 let id = record.question.clone().unwrap();
20154 ask::Questions::open()
20155 .update(&id, |q| {
20156 q.choices.push(crate::github_text::USE_REWRITE.into());
20157 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20158 })
20159 .unwrap();
20160 let message = runner
20161 .guarded_pr_message(&winner, None, true)
20162 .await
20163 .unwrap()
20164 .unwrap();
20165 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20166 assert_eq!(
20167 runner.state.github_text.as_ref().unwrap().choice,
20168 Some(GithubTextChoice::Fallback)
20169 );
20170 }
20171
20172 #[tokio::test]
20173 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20174 crate::run::pin_test_home();
20175 let dir = tempfile::tempdir().unwrap();
20176 let mut runner = runner_at(RunStatus::Gating);
20177 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20178 runner.state.repo = dir.path().to_owned();
20179 runner.state.candidates[0].worktree = dir.path().to_owned();
20180 seed_public_summary(&mut runner);
20181 let winner = runner.state.candidates[0].clone();
20182 assert!(
20183 runner
20184 .guarded_pr_message(&winner, None, true)
20185 .await
20186 .unwrap()
20187 .is_none()
20188 );
20189 let store = ask::Questions::open();
20190 let find = |run: &str| {
20191 store
20192 .list()
20193 .into_iter()
20194 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20195 .unwrap()
20196 };
20197 let run = runner.state.id.clone();
20198 let q = find(&run);
20199 let fp = runner
20200 .state
20201 .github_text
20202 .as_ref()
20203 .unwrap()
20204 .fingerprint
20205 .clone();
20206 let name = crate::github_text::artifact_name(&fp);
20207 let path = crate::run::artifact_path(&runner.state, &name);
20208 assert!(q.detail.contains(&path.display().to_string()));
20209 assert!(q.detail.contains("title-language: title"));
20210 let saved = std::fs::read_to_string(&path).expect("artifact written");
20211 assert!(saved.contains("日本語のタイトル"));
20212 assert!(q.cwd.is_none());
20213 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20214 let say = |id: &str, text: &str, choice: &str| {
20216 store
20217 .update(id, |q| {
20218 q.thread.push(ask::Turn {
20219 who: ask::Who::Operator,
20220 body: text.to_owned(),
20221 at: jiff::Timestamp::now(),
20222 note: None,
20223 });
20224 q.answer(ask::Answer::Choice(choice.to_owned()))
20225 })
20226 .unwrap();
20227 };
20228 say(&q.id, "まだ日本語", "use my text");
20229 assert!(
20230 runner
20231 .guarded_pr_message(&winner, None, true)
20232 .await
20233 .unwrap()
20234 .is_none()
20235 );
20236 let second = find(&run);
20237 assert_ne!(second.id, q.id);
20238 assert!(second.detail.contains("did not pass"));
20239 assert!(second.detail.contains("日本語のタイトル"));
20242 assert!(!second.detail.contains("まだ日本語"));
20243 assert!(second.detail.contains(&path.display().to_string()));
20244 say(&second.id, "fix: retry failed requests", "use my text");
20245 let message = runner
20246 .guarded_pr_message(&winner, None, true)
20247 .await
20248 .unwrap()
20249 .expect("a vetted replacement is posted");
20250 assert_eq!(message.title, "fix: retry failed requests");
20251 assert_eq!(
20252 runner
20253 .state
20254 .github_text
20255 .as_ref()
20256 .unwrap()
20257 .chosen_title
20258 .as_deref(),
20259 Some("fix: retry failed requests")
20260 );
20261 assert!(runner.state.github_text.as_ref().unwrap().resolved);
20262 let again = runner
20264 .guarded_pr_message(&winner, None, true)
20265 .await
20266 .unwrap()
20267 .unwrap();
20268 assert_eq!(again.title, message.title);
20269 assert_eq!(
20270 store
20271 .list()
20272 .into_iter()
20273 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
20274 .count(),
20275 2
20276 );
20277 }
20278
20279 #[test]
20280 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
20281 let mut state = state_with_summary(
20285 "add retries",
20286 "TITLE: fix(web): batch reads\n- reads run.json once",
20287 );
20288 state.config.graph.language = "ja".to_owned();
20289 let m = pr_message(&state, 'A');
20290 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
20291 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
20292
20293 let task = "今回やってほしいこと: results projector を直す";
20296 let mut state = state_with_summary(task, "- no title line");
20297 state.config.graph.language = "ja".to_owned();
20298 let m = pr_message(&state, 'A');
20299 assert_eq!(
20300 m.title,
20301 format!("chore: land candidate A of run {}", state.id)
20302 );
20303 assert!(
20304 m.body.contains(&format!(
20305 "<summary>Original task</summary>\n\n{task}\n\n</details>"
20306 )),
20307 "{}",
20308 m.body
20309 );
20310 }
20311
20312 #[test]
20313 fn pr_message_scrubs_home_paths_and_addresses() {
20314 let state = state_with_summary(
20315 "fix it in /Users/someone/src/x",
20316 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
20317 );
20318 let m = pr_message(&state, 'A');
20319 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
20320 assert!(!m.body.contains(leak), "{}", m.body);
20321 }
20322 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
20323 }
20324
20325 #[test]
20326 fn pr_message_survives_a_task_that_closes_details() {
20327 let state = state_with_summary("a </details> b", "TITLE: fix: x");
20328 let m = pr_message(&state, 'A');
20329 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
20330 }
20331
20332 #[test]
20333 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
20334 let cmd = manual_merge_command(
20335 MergeStyle::Squash,
20336 Path::new("/repo"),
20337 "b",
20338 "fix: \"quoted\" $(x) `y`\n\nbody",
20339 );
20340 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
20341 }
20342
20343 #[test]
20344 fn manual_merge_command_matches_the_configured_style() {
20345 let repo = Path::new("/repo");
20346 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
20347
20348 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
20349 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
20350
20351 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
20352 assert_eq!(
20353 squash,
20354 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
20355 \"Merge magi run 0832 (candidate A)\""
20356 );
20357
20358 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
20359 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
20360 }
20361
20362 #[test]
20363 fn a_nudge_gets_a_quarter_of_the_budget() {
20364 assert_eq!(retry_budget(secs(1200), true), secs(300));
20366 assert_eq!(retry_budget(secs(3600), true), secs(900));
20367 }
20368
20369 #[test]
20370 fn a_resent_prompt_keeps_the_whole_budget() {
20371 assert_eq!(retry_budget(secs(1200), false), secs(1200));
20374 assert_eq!(retry_budget(secs(60), false), secs(60));
20375 }
20376
20377 #[test]
20378 fn the_floor_never_exceeds_the_original_budget() {
20379 assert_eq!(retry_budget(secs(60), true), secs(60));
20383 assert_eq!(retry_budget(secs(480), true), secs(120));
20384 assert_eq!(retry_budget(secs(0), true), secs(0));
20385 }
20386
20387 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
20388 agent::CommandEvidence {
20389 id: "item1".to_owned(),
20390 description: "cargo test".to_owned(),
20391 exit_code,
20392 result_summary: String::new(),
20393 source: "codex".to_owned(),
20394 }
20395 }
20396
20397 #[test]
20398 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
20399 assert!(!has_unconfirmed_command(&[]));
20403 }
20404
20405 #[test]
20406 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
20407 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
20411 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
20412 assert!(!has_unconfirmed_command(&[
20413 evidence(Some(0)),
20414 evidence(Some(101))
20415 ]));
20416 }
20417
20418 #[test]
20419 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
20420 assert!(has_unconfirmed_command(&[
20421 evidence(Some(0)),
20422 evidence(None)
20423 ]));
20424 }
20425
20426 #[test]
20427 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
20428 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20429 assert_eq!(
20430 verified_noop_claim(true, &[], text).as_deref(),
20431 Some("already fixed by b32cfc4, on main.")
20432 );
20433 }
20434
20435 #[test]
20436 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
20437 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20440 assert!(verified_noop_claim(false, &[], text).is_none());
20441 }
20442
20443 #[test]
20444 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
20445 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20446 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
20447 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
20449 }
20450
20451 #[test]
20452 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
20453 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
20454 }
20455
20456 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
20459 runner.state.candidates = shape
20460 .iter()
20461 .enumerate()
20462 .map(|(i, &(empty, verified))| Candidate {
20463 index: i,
20464 label: (b'A' + i as u8) as char,
20465 agent: "sonnet".to_owned(),
20466 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
20467 worktree: PathBuf::from(format!("/wt/{i}")),
20468 summary: String::new(),
20469 stat: String::new(),
20470 files: 0,
20471 commits: 0,
20472 empty,
20473 failed: None,
20474 verified_noop: verified.map(str::to_owned),
20475 duration_ms: 0,
20476 folded: false,
20477 })
20478 .collect();
20479 }
20480
20481 #[test]
20482 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
20483 ask_test_home();
20484 let mut runner = runner_at(RunStatus::Implementing);
20485 set_candidates(
20486 &mut runner,
20487 &[
20488 (true, Some("already on main at b32cfc4")),
20489 (true, Some("same fix, see the existing test")),
20490 ],
20491 );
20492
20493 runner
20494 .after_implement()
20495 .expect("a verified no-op is not an error");
20496
20497 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
20498 }
20499
20500 #[test]
20501 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
20502 ask_test_home();
20503 let mut runner = runner_at(RunStatus::Implementing);
20504 set_candidates(
20508 &mut runner,
20509 &[(true, Some("already on main at b32cfc4")), (true, None)],
20510 );
20511
20512 let err = runner
20513 .after_implement()
20514 .expect_err("an unverified empty candidate must still fail the run");
20515
20516 assert!(
20517 err.to_string().contains("no candidate produced a change"),
20518 "{err}"
20519 );
20520 assert_eq!(runner.state.status, RunStatus::Failed);
20521 }
20522
20523 #[test]
20524 fn after_implement_still_fails_an_ordinary_all_empty_run() {
20525 ask_test_home();
20526 let mut runner = runner_at(RunStatus::Implementing);
20527 set_candidates(&mut runner, &[(true, None), (true, None)]);
20528
20529 let err = runner
20530 .after_implement()
20531 .expect_err("no candidate declared anything; this is an ordinary failure");
20532
20533 assert!(
20534 err.to_string().contains("no candidate produced a change"),
20535 "{err}"
20536 );
20537 assert_eq!(runner.state.status, RunStatus::Failed);
20538 }
20539}
20540
20541#[derive(Debug)]
20544struct GraftRestoreFailed(String);
20545
20546impl std::fmt::Display for GraftRestoreFailed {
20547 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
20548 write!(
20549 f,
20550 "could not restore the winner to its pre-graft commit: {}",
20551 self.0
20552 )
20553 }
20554}
20555
20556impl std::error::Error for GraftRestoreFailed {}
20557
20558#[cfg(test)]
20562mod resource_wait_tests {
20563 use super::*;
20564 use std::collections::BTreeMap;
20565
20566 fn sh(id: &str, script: &str) -> AgentSpec {
20567 AgentSpec {
20568 id: id.to_owned(),
20569 kind: crate::config::AgentKind::Command,
20570 model: None,
20571 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
20572 extra_args: Vec::new(),
20573 env: BTreeMap::new(),
20574 prompt_delivery: None,
20575 max_turn_secs: None,
20576 }
20577 }
20578
20579 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
20580 SeatJob {
20581 seat: SeatState::new("impl-A", &spec.id, 1),
20582 spec,
20583 cwd: dir.to_path_buf(),
20584 prompt: "do it".to_owned(),
20585 timeout,
20586 allow_write: true,
20587 sessions: false,
20588 artifacts: dir.join("art"),
20589 stem: "impl-A".to_owned(),
20590 handover: None,
20591 }
20592 }
20593
20594 fn state(resource_wait_secs: u64) -> RunState {
20595 let mut config = Config::default();
20596 config.graph.timeout_resource_wait = resource_wait_secs;
20597 RunState::new(
20598 PathBuf::from("/repo"),
20599 "main".to_owned(),
20600 "abc1234".to_owned(),
20601 "task".to_owned(),
20602 config,
20603 )
20604 }
20605
20606 fn hold(cache: &Path) -> crate::cache::Guard {
20607 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
20608 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
20609 crate::cache::AcquireOutcome::Acquired(g) => g,
20610 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
20611 }
20612 }
20613
20614 async fn run(
20615 jobs: Vec<SeatJob>,
20616 sem: Arc<Semaphore>,
20617 cache: Option<&Path>,
20618 st: &mut RunState,
20619 ) -> Vec<(usize, SeatState, AgentOutcome)> {
20620 let prompts = Prompts::default();
20621 let ctx = WaveCtx {
20622 run: "r",
20623 node: "implement",
20624 prompts: &prompts,
20625 cache,
20626 round: None,
20627 carry_seats: false,
20628 pauses: None,
20629 };
20630 wave(jobs, sem, &ctx, st, 0).await
20631 }
20632
20633 #[tokio::test]
20634 async fn verification_cache_wait_preserves_the_command_budget() {
20635 crate::run::pin_test_home();
20636 let tmp = tempfile::tempdir().unwrap();
20637 let cache = tmp.path().join("target");
20638 let guard = hold(&cache);
20639 let release = tokio::spawn(async move {
20640 tokio::time::sleep(Duration::from_millis(100)).await;
20641 guard.release();
20642 });
20643 let mut st = state(10);
20644 let budget = Duration::from_millis(50);
20645 let wait = repository_wait(&st, None);
20646 let (outcomes, _) = with_cache_lease(
20647 &mut st,
20648 Some(&cache),
20649 "gate",
20650 "gate",
20651 tmp.path(),
20652 "abc1234",
20653 budget,
20654 "test gate",
20655 &wait,
20656 |_, available| async move {
20657 assert_eq!(
20658 available, budget,
20659 "contention must not shorten verification"
20660 );
20661 (Vec::new(), false, Vec::new())
20662 },
20663 )
20664 .await
20665 .unwrap();
20666 release.await.unwrap();
20667 assert!(outcomes.is_empty(), "{outcomes:?}");
20668 assert!(
20669 st.events
20670 .iter()
20671 .any(|event| event.message.contains("waiting for"))
20672 );
20673 assert!(st.resource_wait.is_none());
20674 hold(&cache).release();
20676 }
20677
20678 #[tokio::test]
20679 async fn verification_cache_wait_expires_without_starting_commands() {
20680 crate::run::pin_test_home();
20681 let tmp = tempfile::tempdir().unwrap();
20682 let cache = tmp.path().join("target");
20683 let _held = hold(&cache);
20684 let mut st = state(0);
20685 let wait = repository_wait(&st, None);
20686 let (outcomes, _) = with_cache_lease(
20687 &mut st,
20688 Some(&cache),
20689 "gate",
20690 "gate",
20691 tmp.path(),
20692 "abc1234",
20693 Duration::from_secs(60),
20694 "test gate",
20695 &wait,
20696 |_, _| async { panic!("commands must not start without the cache lease") },
20697 )
20698 .await
20699 .unwrap();
20700 assert_eq!(outcomes.len(), 1);
20701 assert!(outcomes[0].resource_blocked);
20702 assert!(st.resource_wait.is_none());
20703 assert!(matches!(
20704 st.resource_block.unwrap().cause,
20705 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
20706 ));
20707 }
20708
20709 #[tokio::test]
20710 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
20711 crate::run::pin_test_home();
20712 let tmp = tempfile::tempdir().unwrap();
20713 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
20714 let cache = tmp.path().join("cache");
20715 let mut state = state(21600);
20716 let wait = repository_wait(&state, None);
20717 let (outcomes, _) = with_cache_lease(
20718 &mut state,
20719 Some(&cache),
20720 "gate",
20721 "gate",
20722 tmp.path(),
20723 "head",
20724 Duration::from_secs(60),
20725 "gate",
20726 &wait,
20727 |_, _| async { panic!("freshness failure must not start verification") },
20728 )
20729 .await
20730 .unwrap();
20731 assert_eq!(outcomes.len(), 1);
20732 assert!(outcomes[0].resource_blocked);
20733 let block = state.resource_block.unwrap();
20734 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
20735 assert!(!block.description().contains("timed out"));
20736 }
20737
20738 #[tokio::test]
20739 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
20740 crate::run::pin_test_home();
20741 let tmp = tempfile::tempdir().unwrap();
20742 let cache = tmp.path().join("cache");
20743 let _held = hold(&cache);
20744 let mut state = state(21600);
20745 let pause = Pause::new();
20746 let interrupt = Pause::new();
20747 let cause = ParkCause::Operator;
20748 pause.park_for(cause.clone());
20749 let prompts = Prompts::default();
20750 let ctx = WaveCtx {
20751 run: "test",
20752 node: "fix",
20753 prompts: &prompts,
20754 cache: Some(&cache),
20755 round: None,
20756 carry_seats: false,
20757 pauses: Some((&pause, &interrupt)),
20758 };
20759 let result = wave(
20760 vec![job(
20761 sh("writer", "exit 99"),
20762 tmp.path(),
20763 Duration::from_secs(1),
20764 )],
20765 Arc::new(Semaphore::new(1)),
20766 &ctx,
20767 &mut state,
20768 0,
20769 )
20770 .await;
20771 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
20772 assert_eq!(result[0].1.turns, 0);
20773 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20774 assert_eq!(
20775 state.resource_block.unwrap().cause,
20776 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
20777 );
20778 assert!(state.active.is_empty());
20779 }
20780
20781 #[tokio::test]
20782 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
20783 crate::run::pin_test_home();
20784 let tmp = tempfile::tempdir().unwrap();
20785 let cache = tmp.path().join("target-a");
20786 let guard = hold(&cache);
20787 tokio::spawn(async move {
20789 tokio::time::sleep(Duration::from_millis(1500)).await;
20790 guard.release();
20791 });
20792 let mut st = state(60);
20793 let results = run(
20794 vec![job(
20795 sh("alpha", "echo ok"),
20796 tmp.path(),
20797 Duration::from_secs(1),
20798 )],
20799 Arc::new(Semaphore::new(1)),
20800 Some(&cache),
20801 &mut st,
20802 )
20803 .await;
20804 assert!(
20805 matches!(results[0].2, AgentOutcome::Ok(_)),
20806 "the wait must not be charged to the agent"
20807 );
20808 assert!(st.resource_block.is_none());
20809 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20810 assert!(st.resource_wait.is_none() && st.active.is_empty());
20811 }
20812
20813 #[tokio::test]
20814 #[cfg(unix)]
20815 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20816 use std::os::fd::AsRawFd as _;
20817 crate::run::pin_test_home();
20818 let tmp = tempfile::tempdir().unwrap();
20819 crate::git::git(tmp.path(), &["init"]).await.unwrap();
20820 let common = tmp.path().join(".git");
20821 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20822 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20823 assert_eq!(
20824 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20825 0
20826 );
20827 let mut st = state(1);
20828 st.config.coordination = crate::config::test_coordination();
20829 let mut reader = job(
20830 sh("reader", "echo must-not-run"),
20831 tmp.path(),
20832 Duration::from_secs(30),
20833 );
20834 reader.allow_write = false;
20835 let results = run(
20836 vec![reader.clone()],
20837 Arc::new(Semaphore::new(1)),
20838 None,
20839 &mut st,
20840 )
20841 .await;
20842 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20843 assert_eq!(results[0].1.turns, 0);
20844 assert!(FailClass::of(&results[0].2).is_none());
20845 assert_eq!(
20846 st.resource_block.as_ref().unwrap().kind,
20847 "repository coordination"
20848 );
20849 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20850 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20851 drop(external);
20852 let editor = crate::repo_lock::SeatLease::for_checkout(
20853 tmp.path(),
20854 true,
20855 crate::config::test_coordination().as_ref(),
20856 )
20857 .await
20858 .unwrap();
20859 let pause = Pause::new();
20860 let interrupt = Pause::new();
20861 let prompts = Prompts::default();
20862 let ctx = WaveCtx {
20863 run: "r",
20864 node: "review",
20865 prompts: &prompts,
20866 cache: None,
20867 round: None,
20868 carry_seats: false,
20869 pauses: Some((&pause, &interrupt)),
20870 };
20871 let (results, ()) = tokio::join!(
20872 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20873 async {
20874 tokio::time::sleep(Duration::from_millis(50)).await;
20875 pause.park();
20876 }
20877 );
20878 assert!(
20879 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20880 );
20881 assert_eq!(results[0].1.turns, 0);
20882 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20883 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20884 drop(editor);
20885 }
20886
20887 #[tokio::test]
20888 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20889 crate::run::pin_test_home();
20890 let tmp = tempfile::tempdir().unwrap();
20891 let cache = tmp.path().join("target-b");
20892 let _held = hold(&cache);
20893 let mut st = state(1);
20894 let results = run(
20895 vec![job(
20896 sh("alpha", "echo ok"),
20897 tmp.path(),
20898 Duration::from_secs(1),
20899 )],
20900 Arc::new(Semaphore::new(1)),
20901 Some(&cache),
20902 &mut st,
20903 )
20904 .await;
20905 let (_, seat, out) = &results[0];
20906 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
20907 assert!(wave_blocked(&results));
20908 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
20909 assert_eq!(seat.turns, 0, "nothing was asked");
20910 assert!(
20911 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
20912 "no agent was spawned"
20913 );
20914 assert!(FailClass::of(out).is_none(), "not a failure class");
20915 let block = st.resource_block.as_ref().expect("recorded for the resume");
20916 assert_eq!(block.kind, "shared build cache");
20917 assert!(matches!(
20918 block.cause,
20919 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
20920 ));
20921 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20922 assert!(st.active.is_empty());
20923 }
20924
20925 #[tokio::test]
20926 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
20927 crate::run::pin_test_home();
20928 let tmp = tempfile::tempdir().unwrap();
20929 let mut st = state(1);
20930 let results = run(
20931 vec![job(
20932 sh("alpha", "echo ok"),
20933 tmp.path(),
20934 Duration::from_secs(1),
20935 )],
20936 Arc::new(Semaphore::new(0)),
20937 None,
20938 &mut st,
20939 )
20940 .await;
20941 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20942 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
20943 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
20944 }
20945
20946 #[tokio::test]
20947 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
20948 crate::run::pin_test_home();
20949 let tmp = tempfile::tempdir().unwrap();
20950 let mut st = state(60);
20951 let results = run(
20952 vec![job(
20953 sh("alpha", "sleep 30"),
20954 tmp.path(),
20955 Duration::from_secs(1),
20956 )],
20957 Arc::new(Semaphore::new(1)),
20958 None,
20959 &mut st,
20960 )
20961 .await;
20962 let out = &results[0].2;
20963 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
20964 let class = FailClass::of(out).expect("a failure");
20965 assert_eq!(class, FailClass::Timeout);
20966 assert!(should_hand_over(None, &class), "existing handover policy");
20967 assert!(st.resource_block.is_none());
20968 }
20969}