1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530enum FixStep {
532 Resource,
533 Recorded { report_lost: bool },
534}
535
536struct FixClaim {
553 path: PathBuf,
554}
555
556impl FixClaim {
557 fn acquire(dir: &Path) -> Result<Self> {
558 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559 let path = dir.join("fix.lock");
560 let retry = crate::lock_file::Retry::new(None);
561 match Self::create(&path, retry) {
562 Ok(claim) => Ok(claim),
563 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564 if Self::reclaim_if_dead(&path) {
565 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566 } else {
567 bail!(
568 "another repair or `magi fix` is already running for this run ({} exists)",
569 path.display()
570 )
571 }
572 }
573 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574 }
575 }
576
577 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578 let mut f = retry.create(path)?;
579 use std::io::Write as _;
580 writeln!(f, "{}", std::process::id())?;
582 Ok(Self {
583 path: path.to_owned(),
584 })
585 }
586
587 fn reclaim_if_dead(path: &Path) -> bool {
591 let dead = std::fs::read_to_string(path)
592 .ok()
593 .and_then(|body| body.trim().parse::<u32>().ok())
594 .is_some_and(|pid| !crate::proc::pid_alive(pid));
595 dead && std::fs::remove_file(path).is_ok()
596 }
597}
598
599impl Drop for FixClaim {
600 fn drop(&mut self) {
601 let _ = std::fs::remove_file(&self.path);
602 }
603}
604
605struct DriverExecution<'a> {
608 runner: &'a mut Runner,
609 driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613 fn drop(&mut self) {
614 self.driver.release_local(&mut self.runner.state);
615 }
616}
617
618impl Runner {
619 pub async fn start(
621 repo: &Path,
622 instruction: String,
623 config: Config,
624 origin: Origin,
625 ) -> Result<Self> {
626 Self::start_naming(repo, instruction, "", config, origin).await
627 }
628
629 pub async fn start_naming(
632 repo: &Path,
633 instruction: String,
634 _also_scan: &str,
635 config: Config,
636 origin: Origin,
637 ) -> Result<Self> {
638 let repo = git::toplevel(repo).await?;
639 let missing = agent::missing_programs(&config.agents);
640 if !missing.is_empty() {
641 bail!(
642 "these agent programs are not on PATH: {}. Fix the roster in \
643 magi.toml or install them.",
644 missing.join(", ")
645 );
646 }
647 let base_branch =
648 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649 .await?;
650 let fetched_base = resolve_base(
651 &repo,
652 &base_branch,
653 &config.merge.remote,
654 config.coordination.as_ref(),
655 )
656 .await?;
657 let base_commit = fetched_base.commit.clone();
658 let roles = config.resolve_roles()?;
659 let max_parallel = config.graph.max_parallel.max(1);
660 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661 state.fetched_base = Some(fetched_base);
665 state.origin = Some(origin);
666 state.reserve()?;
667 state.event("start", format!("run {} created", state.id));
668 state.save()?;
669 Ok(Self {
670 state,
671 roles,
672 sem: Arc::new(Semaphore::new(max_parallel)),
673 pause: Pause::new(),
674 interrupt: Pause::new(),
675 })
676 }
677
678 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692 Self::review_taking_over(repo, branch, config, None, origin).await
693 }
694
695 pub async fn review_taking_over(
702 repo: &Path,
703 branch: &str,
704 config: Config,
705 takeover: Option<crate::handover::Takeover>,
706 origin: Origin,
707 ) -> Result<Self> {
708 let repo = git::toplevel(repo).await?;
709 let missing = agent::missing_programs(&config.agents);
710 if !missing.is_empty() {
711 bail!(
712 "these agent programs are not on PATH: {}. Fix the roster in \
713 magi.toml or install them.",
714 missing.join(", ")
715 );
716 }
717 let base_branch =
718 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719 .await?;
720 if base_branch == branch {
721 bail!("`{branch}` is the base branch; there is nothing to review against");
722 }
723 let fetched_base = resolve_base(
724 &repo,
725 &base_branch,
726 &config.merge.remote,
727 config.coordination.as_ref(),
728 )
729 .await?;
730 let base_commit = fetched_base.commit.clone();
731
732 let roles = config.resolve_roles()?;
733 let max_parallel = config.graph.max_parallel.max(1);
734 let mut state = RunState::new(
735 repo.clone(),
736 base_branch,
737 base_commit.clone(),
738 String::new(),
739 config,
740 );
741 state.fetched_base = Some(fetched_base);
742 state.origin = Some(origin);
743 state.reserve()?;
744
745 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752 earlier: Vec::new(),
753 home: crate::run::home(),
754 choice: None,
755 task_instruction: None,
756 });
757 let wait = repository_wait(&state, None);
758 let coordination_config = state.config.coordination.clone();
759 let coordination =
760 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761 .await?;
762 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763 if let Some(released) = &released {
764 state.event(
765 "release",
766 format!(
767 "took `{branch}` over from run {}: its worktree was released: {}",
768 crate::run::short_of(&released.old_id),
769 released.audit
770 ),
771 );
772 }
773 if let Some(choice) = takeover.choice.as_ref()
777 && let Err(e) =
778 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779 .await
780 {
781 if let Some(released) = &released {
782 released.restore(&repo, branch).await;
783 }
784 return Err(e.context("applying the owner's answer about the diverged branch"));
785 }
786 for id in takeover.earlier.iter().rev() {
792 if let Ok(old) = RunState::load_under(id, &takeover.home)
793 && let Some(head) = old.pr_head.clone()
794 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795 {
796 state.pr_head = Some(head);
797 break;
798 }
799 }
800 drop(coordination);
801 let opened = Self::open_review(
802 &repo,
803 branch,
804 state,
805 roles,
806 max_parallel,
807 base_commit,
808 &takeover,
809 )
810 .await;
811 if opened.is_err()
812 && let Some(released) = &released
813 {
814 let _coordination = crate::repo_lock::SeatLease::operation_with(
815 &repo,
816 coordination_config.as_ref(),
817 &wait,
818 )
819 .await?;
820 released.restore(&repo, branch).await;
821 }
822 opened
823 }
824
825 pub async fn recover_pr(
827 repo: &Path,
828 task: &str,
829 selection: crate::pr_recovery::Selection,
830 mut config: Config,
831 ) -> Result<Self> {
832 selection.configure(&mut config)?;
833 selection.check_task(task, &selection.instruction)?;
834 let base_branch =
835 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836 .await?;
837 let fetched_base = resolve_base(
838 repo,
839 &base_branch,
840 &config.merge.remote,
841 config.coordination.as_ref(),
842 )
843 .await?;
844 let base_commit = fetched_base.commit.clone();
845 selection
846 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847 .await?;
848 Box::pin(selection.verify_source(repo)).await?;
849 if let Some(proof) = &selection.source_proof {
850 anyhow::ensure!(
851 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852 "source rebase base is outside the fetched landing base"
853 );
854 }
855 let roles = config.resolve_roles()?;
856 let max_parallel = config.graph.max_parallel.max(1);
857 let mut state = RunState::new(
858 repo.to_path_buf(),
859 base_branch,
860 base_commit.clone(),
861 selection.instruction.clone(),
862 config,
863 );
864 state.fetched_base = Some(fetched_base);
865 state.origin = Some(crate::run::Origin::queue(task));
866 state.pr_head = Some(selection.branch.clone());
867 state.pr_recovery = Some(selection.clone());
868 state.reserve()?;
869 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870 state.discard_reserved();
872 return Err(e);
873 }
874 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875 }
876
877 async fn open_selected(
878 repo: &Path,
879 state: RunState,
880 roles: ResolvedRoles,
881 max_parallel: usize,
882 base_commit: String,
883 selection: crate::pr_recovery::Selection,
884 ) -> Result<Self> {
885 let mut state = state;
886 let branch = state.branch_for('A');
887 let wait = repository_wait(&state, None);
888 let prepared = async {
889 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890 git::git(repo, &["branch", &branch, selection.source()]).await?;
891 drop(coordination);
892 anyhow::Ok(())
893 }
894 .await;
895 if let Err(e) = prepared {
896 state.discard_reserved();
899 return Err(e);
900 }
901 state.event(
902 "recovery",
903 format!(
904 "selected implementation {} for {} (public lease {})",
905 selection.source(),
906 selection.url,
907 selection.head
908 ),
909 );
910 let takeover = crate::handover::Takeover {
911 earlier: Vec::new(),
912 home: crate::run::home(),
913 choice: None,
914 task_instruction: Some(selection.instruction.clone()),
915 };
916 if let Err(e) = state.save() {
917 state.discard_reserved();
918 return Err(e);
919 }
920 let id = state.id.clone();
921 let opened = Self::open_review(
922 repo,
923 &branch,
924 state,
925 roles,
926 max_parallel,
927 base_commit,
928 &takeover,
929 )
930 .await;
931 match opened {
932 Ok(runner) => Ok(runner),
933 Err(e) => {
934 if let Ok(mut partial) = RunState::load(&id) {
935 partial.status = RunStatus::Blocked;
936 partial.driver_exited = true;
937 partial.event(
938 "recovery",
939 format!("startup failed; retained source and candidate {branch}: {e:#}"),
940 );
941 partial.save()?;
942 }
943 Err(crate::pr_recovery::StartupFailed {
944 run: id,
945 cause: format!("{e:#}"),
946 }
947 .into())
948 }
949 }
950 }
951
952 async fn inherit(
957 repo: &Path,
958 branch: &str,
959 base_commit: &str,
960 takeover: &crate::handover::Takeover,
961 ) -> Option<crate::recovery::RecoveryRecord> {
962 use crate::recovery::{Relation, assess};
963 let newest = takeover.earlier.last()?;
964 let mut chosen = None;
965 for id in takeover.earlier.iter().rev() {
966 if let Ok(old) = RunState::load_under(id, &takeover.home)
967 && !old.reviews.is_empty()
968 && (old.winner().is_some_and(|w| w.branch == branch)
969 || old.pr_head.as_deref() == Some(branch))
970 {
971 chosen = Some(old);
972 break;
973 }
974 }
975 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976 .await
977 .unwrap_or_default();
978 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979 Some(last) if last.head == tip => Relation::Same,
980 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981 _ => Relation::Diverged,
982 };
983 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984 Some(assess(from, chosen.as_ref(), base_commit, relation))
985 }
986
987 async fn open_review(
990 repo: &Path,
991 branch: &str,
992 mut state: RunState,
993 roles: ResolvedRoles,
994 max_parallel: usize,
995 base_commit: String,
996 takeover: &crate::handover::Takeover,
997 ) -> Result<Self> {
998 let wait = repository_wait(&state, None);
999 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000 if state.pr_recovery.is_none() {
1001 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002 }
1003 let start = review_base(
1006 repo,
1007 &state.config.merge.remote,
1008 &state.base_branch,
1009 &base_commit,
1010 branch,
1011 )
1012 .await;
1013 let log = git::log_oneline(repo, &start, branch)
1014 .await
1015 .unwrap_or_default();
1016 let log_text = if log.trim().is_empty() {
1017 "(no commit messages)"
1018 } else {
1019 log.trim()
1020 };
1021 let mut instruction = format!(
1022 "Review the work already on branch `{branch}`. There is no task \
1023 statement: what the change claims to do is whatever its commits \
1024 say.\n\n{log_text}"
1025 );
1026 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030 if let Some(task) = takeover
1031 .task_instruction
1032 .as_deref()
1033 .filter(|t| !t.trim().is_empty())
1034 && rec.reviewed_head.is_some()
1035 {
1036 instruction = format!(
1037 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038 task.trim()
1039 );
1040 rec.criteria_inherited = true;
1041 }
1042 state.event("recovery", rec.summary());
1043 state.recovery = Some(rec);
1044 }
1045 if let Some(selected) = &state.pr_recovery {
1046 instruction = selected.instruction.clone();
1047 }
1048 state.task_instruction = takeover.task_instruction.clone();
1049 state.instruction = instruction;
1050 state.reviewed_commits = Some(
1051 git::subjects(repo, &start, branch)
1052 .await
1053 .unwrap_or_default(),
1054 );
1055
1056 for id in &takeover.earlier {
1059 if let Ok(old) = RunState::load_under(id, &takeover.home)
1060 && old.repo == state.repo
1061 && (old.pr_head.as_deref() == Some(branch)
1062 || old.winner().is_some_and(|w| w.branch == branch))
1063 {
1064 for hold in old.branch_owner_holds {
1065 if !state.branch_owner_holds.contains(&hold) {
1066 state.event(
1067 "approval-policy",
1068 format!(
1069 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070 hold.control.key(),
1071 hold.file,
1072 hold.head
1073 ),
1074 );
1075 state.branch_owner_holds.push(hold);
1076 }
1077 }
1078 }
1079 }
1080 let reviewed_head = git::rev_parse(repo, branch).await?;
1081 state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083 let worktree = state.worktree_root().join("under-review");
1086 if let Some(parent) = worktree.parent() {
1087 tokio::fs::create_dir_all(parent).await.ok();
1088 }
1089 let path = worktree.to_string_lossy().to_string();
1090 git::git(repo, &["worktree", "add", &path, branch])
1091 .await
1092 .with_context(|| {
1093 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094 })?;
1095
1096 let setup = match ensure_setup_config(&mut state, repo).await {
1097 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098 Err(e) => Err(e),
1099 };
1100 if let Err(e) = setup {
1101 git::worktree_remove_clean(repo, &worktree).await.ok();
1102 return Err(e);
1103 }
1104 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105 .await
1106 .unwrap_or(0);
1107 if commits == 0 {
1108 git::worktree_remove_clean(repo, &worktree).await.ok();
1109 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110 }
1111 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112 .await
1113 .map(|f| f.len())
1114 .unwrap_or(0);
1115 if files == 0
1116 && let (Ok(head_tree), Ok(base_tree)) = (
1117 git::tree_of(&worktree, "HEAD").await,
1118 git::tree_of(&worktree, &base_commit).await,
1119 )
1120 && head_tree == base_tree
1121 {
1122 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123 git::worktree_remove_clean(repo, &worktree).await.ok();
1124 bail!(
1125 "`{branch}` at {} has a tree identical to base {}; this usually means \
1126 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127 against `{}/{branch}`) rather than an empty change",
1128 short(&head),
1129 short(&base_commit),
1130 state.config.merge.remote
1131 );
1132 }
1133 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134 .await
1135 .unwrap_or_default();
1136
1137 state.candidates.push(Candidate {
1138 index: 0,
1139 label: 'A',
1140 agent: EXISTING_BRANCH.to_owned(),
1143 branch: branch.to_owned(),
1144 worktree,
1145 summary: String::new(),
1146 stat,
1147 files,
1148 commits,
1149 empty: false,
1150 failed: None,
1151 verified_noop: None,
1152 duration_ms: 0,
1153 folded: false,
1154 });
1155 state.tally = Some(Tally {
1156 first_choice: BTreeMap::from([('A', 0)]),
1157 borda: BTreeMap::new(),
1158 winner: 'A',
1159 rankings: 0,
1160 unanimous_initial: false,
1161 deliberated: false,
1162 changed_votes: 0,
1163 unanimous_final: false,
1164 tie_break: None,
1165 judges: 0,
1169 present: 0,
1170 quorum: 0,
1171 met_quorum: true,
1172 uncontested: Some("review-only run: nothing competed".to_owned()),
1173 });
1174 state.status = RunStatus::Reviewing;
1175 state.prep_complete = Some(true);
1176 state.event(
1177 "start",
1178 format!(
1179 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180 state.id
1181 ),
1182 );
1183 state.save()?;
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub fn resume(id: &str) -> Result<Self> {
1195 let state = RunState::load(id)?;
1196 crate::pr_head::ensure_owner(&state)?;
1197 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198 bail!(
1199 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200 );
1201 }
1202 if let Some(to) = &state.released_to {
1203 bail!(
1204 "run {} cannot be resumed: its worktree was released to run {}",
1205 state.short(),
1206 crate::run::short_of(to)
1207 );
1208 }
1209 let roles = state.config.resolve_roles()?;
1210 if let Some(id) = &state.config.graph.selected_implementer
1211 && state.candidates.iter().any(|c| c.agent != *id)
1212 {
1213 bail!(
1214 "replacement implementer pin does not match the saved candidates; task must stay held"
1215 );
1216 }
1217 let max_parallel = state.config.graph.max_parallel.max(1);
1218 Ok(Self {
1219 state,
1220 roles,
1221 sem: Arc::new(Semaphore::new(max_parallel)),
1222 pause: Pause::new(),
1223 interrupt: Pause::new(),
1224 })
1225 }
1226
1227 pub async fn execute(&mut self) -> Result<()> {
1234 if let Some(selection) = &self.state.pr_recovery
1235 && (selection.source_proof.is_some() || selection.source_run.is_some())
1236 {
1237 Box::pin(selection.verify_source(&self.state.repo)).await?;
1238 }
1239 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242 || self.state.land_approval.is_some()
1243 || self.state.merge_decision.is_some()
1244 || !self.state.merge_repairs.is_empty()
1245 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246 {
1247 let claim = FixClaim::acquire(&self.state.dir())?;
1248 let disk = RunState::load(&self.state.id)?;
1249 let checkpoint = |s: &RunState| {
1250 (
1251 s.merge_repairs.len(),
1252 s.merge_repairs
1253 .last()
1254 .map(|r| (r.question.clone(), r.phase)),
1255 s.land_approval
1256 .as_ref()
1257 .map(|a| (a.question.clone(), a.head.clone())),
1258 )
1259 };
1260 if checkpoint(&disk) != checkpoint(&self.state) {
1261 bail!(
1262 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263 );
1264 }
1265 Some(claim)
1266 } else {
1267 None
1268 };
1269 let mut execution = self.claim_driver().await?;
1270 let runner = &mut *execution.runner;
1271 let wait = runner.repository_wait();
1272 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275 let result = match result {
1276 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277 record_repository_block(&mut runner.state, &e);
1278 runner.park_here()?;
1279 Ok(())
1280 }
1281 other => other,
1282 };
1283 execution.driver.finish(&mut runner.state).await?;
1284 let ended = if result.is_err() {
1285 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1286 } else {
1287 crate::notices::run_ended(&runner.state)
1288 };
1289 if let Some(notice) = ended {
1290 crate::notices::raise(notice);
1291 }
1292 result
1293 }
1294
1295 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1296 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1297 }
1298
1299 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1300 let wait = self.repository_wait();
1301 coordinate_operation(&mut self.state, cwd, &wait).await
1302 }
1303
1304 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1306 let owner = match crate::pr_head::owner_lock_wait(
1307 &crate::run::home(),
1308 &self.state.id,
1309 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1310 )
1311 .await
1312 {
1313 Ok(owner) => owner,
1314 Err(error) => {
1315 let detail = format!("worktree maintenance or ownership update: {error:#}");
1316 self.state.resource_block = Some(crate::run::ResourceBlock {
1320 cause: crate::run::ResourceBlockCause::Other,
1321 node: "start".into(),
1322 kind: "worktree maintenance".into(),
1323 detail: detail.clone(),
1324 at: Timestamp::now(),
1325 });
1326 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1327 }
1328 };
1329 let disk = RunState::load(&self.state.id)?;
1330 crate::pr_head::ensure_owner(&disk)?;
1331 let claimed =
1332 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1333 anyhow::ensure!(
1338 disk.liveness(claimed) != crate::run::Liveness::Live,
1339 "run already has a live driver; wait for its execution to end"
1340 );
1341 if let Some(to) = &disk.released_to {
1342 bail!(
1343 "run {} cannot continue: its worktree was released to run {}",
1344 self.state.short(),
1345 crate::run::short_of(to)
1346 );
1347 }
1348 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1349 let mut execution = DriverExecution {
1350 runner: self,
1351 driver,
1352 };
1353 let _owner = owner;
1356 let state = &mut execution.runner.state;
1357 state.parked = false;
1362 state.resource_wait = None;
1363 state.resource_block = None;
1364 state.clear_active();
1371 let pid = std::process::id();
1386 state.driver_pid = Some(pid);
1387 state.driver_started_at = crate::proc::process_started_at(pid);
1388 state.driver_exited = false;
1389 state.driver_claimed = Some(true);
1390 state.begin_pass();
1391 execution.driver.bind(state);
1392 state.save()?;
1393 execution.driver.record(state)?;
1394 Ok(execution)
1395 }
1396
1397 async fn execute_graph(&mut self) -> Result<()> {
1398 self.state.configure_recovery_resume()?;
1399 self.persist_candidates().await?;
1402 if self
1403 .state
1404 .merge_decision
1405 .as_ref()
1406 .is_some_and(|decision| !decision.closed)
1407 && self.state.status != RunStatus::Landing
1408 {
1409 if crate::merge_decision::resumable_hold(&self.state) {
1410 self.state.merge = Some(MergeOutcome {
1411 mode: MergeMode::Pr,
1412 ok: false,
1413 detail: self
1414 .state
1415 .merge_decision
1416 .as_ref()
1417 .unwrap()
1418 .target
1419 .pr
1420 .clone(),
1421 empty: false,
1422 });
1423 self.run_land().await?;
1424 self.settle_questions();
1425 return Ok(());
1426 }
1427 bail!(
1428 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1429 );
1430 }
1431 if self.state.config.graph.selected_implementer.is_some()
1432 && self.state.status == RunStatus::Blocked
1433 && self.state.tally.is_none()
1434 && self.state.candidate_preservation.is_empty()
1435 && self.state.interrupted_implementations.is_empty()
1436 {
1437 bail!(
1438 "selected implementation is held; explicitly release the owning task to retry it"
1439 );
1440 }
1441
1442 if self.state.merge_repairs.last().is_some_and(|r| {
1443 matches!(
1444 r.phase,
1445 crate::run::MergeRepairPhase::Requested
1446 | crate::run::MergeRepairPhase::Fixing
1447 | crate::run::MergeRepairPhase::Reviewing
1448 )
1449 }) {
1450 if self.repair_merge_review().await? {
1451 self.run_land().await?;
1452 }
1453 self.settle_questions();
1454 return Ok(());
1455 }
1456 if self.state.pr_publication_pending {
1459 self.merge().await?;
1460 if self.resource_halt()? {
1461 return Ok(());
1462 }
1463 self.state.save()?;
1464 self.settle_questions();
1465 return Ok(());
1466 }
1467 if self.state.status == RunStatus::Stalled {
1480 if self.recover_stall().await? {
1481 self.finish_after_tally().await?;
1482 } else {
1483 self.state.save()?;
1485 }
1486 return Ok(());
1487 }
1488 if self.state.status == RunStatus::Landing
1498 || crate::handover::landing_checkpoint(&self.state)
1499 {
1500 self.state.land_sync_pending = true;
1501 self.state.save()?;
1502 self.run_land().await?;
1503 self.settle_questions();
1508 return Ok(());
1509 }
1510 if self.state.github_text.is_some()
1515 && self.state.merge.is_none()
1516 && self.state.status == RunStatus::Gating
1517 {
1518 self.merge().await?;
1519 if self.resource_halt()? {
1520 return Ok(());
1521 }
1522 self.state.save()?;
1523 return Ok(());
1524 }
1525 if let Err(e) = self.prep().await {
1526 if e.is::<crate::repo_lock::ResourceBlocked>() {
1527 return Err(e);
1528 }
1529 self.state.status = RunStatus::Blocked;
1530 self.state
1531 .prep_failure
1532 .get_or_insert_with(|| format!("{e:#}"));
1533 self.state
1534 .event("prep", format!("preparation stopped: {e:#}"));
1535 self.state.save()?;
1536 return Err(e);
1537 }
1538 if self.park_here()? {
1539 return Ok(());
1540 }
1541 self.advise().await?;
1542 if self.park_here()? {
1543 return Ok(());
1544 }
1545 self.implement().await?;
1546 if self.park_here()? {
1547 return Ok(());
1548 }
1549 if self.state.status == RunStatus::VerifiedNoop {
1553 return Ok(());
1554 }
1555 self.judge().await?;
1556 if self.park_here()? {
1557 return Ok(());
1558 }
1559 self.deliberate().await?;
1560 if self.park_here()? {
1561 return Ok(());
1562 }
1563 self.vote().await?;
1564 if self.park_here()? {
1565 return Ok(());
1566 }
1567 self.tally()?;
1568 if self.state.status == RunStatus::Stalled {
1573 self.state.save()?;
1577 return Ok(());
1578 }
1579 self.finish_after_tally().await?;
1580 Ok(())
1581 }
1582
1583 fn stop_for_resource(&mut self) -> Result<()> {
1588 if self.pause.parked() || self.interrupt.parked() {
1591 self.park_here()?;
1592 return Ok(());
1593 }
1594 if self.state.resource_block.is_some() {
1595 self.state.status = RunStatus::Blocked;
1596 }
1597 self.state.save()
1598 }
1599
1600 fn resource_halt(&mut self) -> Result<bool> {
1602 if self.state.resource_block.is_none() {
1603 return Ok(false);
1604 }
1605 self.stop_for_resource()?;
1606 Ok(true)
1607 }
1608
1609 fn park_here(&mut self) -> Result<bool> {
1616 let parking = self.pause.parked() || self.interrupt.parked();
1617 if !parking && self.state.resource_block.is_some() {
1618 self.stop_for_resource()?;
1619 return Ok(true);
1620 }
1621 if !self.pause.parked() && !self.interrupt.parked() {
1626 return Ok(false);
1627 }
1628 if self.state.parked {
1629 return Ok(true);
1630 }
1631 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1632 if let Some(block) = &mut self.state.resource_block
1633 && matches!(
1634 block.cause,
1635 crate::run::ResourceBlockCause::Cancelled { .. }
1636 )
1637 {
1638 block.cause = crate::run::ResourceBlockCause::Cancelled {
1639 park: cause.clone(),
1640 };
1641 }
1642 let why = match cause.as_ref().map(ParkCause::label) {
1643 Some(reason) => format!(
1644 "parked after `{}` ({reason}) — resume to carry on from here",
1645 self.state.status.as_str()
1646 ),
1647 None => format!(
1648 "parked after `{}` — resume to carry on from here",
1649 self.state.status.as_str()
1650 ),
1651 };
1652 self.state.event("park", why);
1653 self.state
1654 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1655 self.state.parked = true;
1656 self.state.save()?;
1657 Ok(true)
1658 }
1659
1660 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1662 if self.state.prep_failure.is_none() {
1663 return Ok(());
1664 }
1665 let failed_at = self
1666 .state
1667 .events
1668 .iter()
1669 .rev()
1670 .find(|event| event.node == "prep")
1671 .map_or(self.state.updated_at, |event| event.at);
1672 anyhow::ensure!(
1673 approval_at > failed_at,
1674 "preparation retry requires a new explicit release after the failure"
1675 );
1676 self.state.prep_failure = None;
1677 self.state.prep_complete = Some(false);
1678 self.state
1679 .event("prep-retry", "operator approved retry of preparation");
1680 Ok(())
1681 }
1682
1683 pub fn on_pause(&mut self, pause: Pause) {
1685 self.pause = pause;
1686 }
1687
1688 pub fn watch_interrupt(&mut self, pause: Pause) {
1694 self.interrupt = pause;
1695 }
1696
1697 fn settle_questions(&mut self) {
1717 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1718 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1719 }
1720 }
1721
1722 async fn finish_after_tally(&mut self) -> Result<()> {
1725 self.graft().await?;
1728 if self.resource_halt()? {
1729 return Ok(());
1730 }
1731 self.fold_losers().await?;
1732 self.sync_to_base().await?;
1737 if self.resource_halt()? {
1738 return Ok(());
1739 }
1740 if self.state.status == RunStatus::AlreadyInBase {
1741 return Ok(());
1742 }
1743 self.review_loop().await?;
1744 if self.resource_halt()? {
1745 return Ok(());
1746 }
1747 self.sync_to_base().await?;
1748 if self.resource_halt()? {
1749 return Ok(());
1750 }
1751 if self.state.status == RunStatus::AlreadyInBase {
1752 return Ok(());
1753 }
1754 self.gate().await?;
1755 self.merge().await?;
1756 if self.resource_halt()? {
1757 return Ok(());
1758 }
1759 self.state.save()?;
1760 Ok(())
1761 }
1762
1763 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1770 let result = match ensure_setup_config(&mut self.state, repo).await {
1771 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1772 Err(e) => Err(e),
1773 };
1774 if let Err(e) = result {
1775 git::worktree_remove_clean(repo, wt).await.ok();
1776 if in_prep {
1777 self.rollback_prep(repo).await;
1778 }
1779 self.state.status = RunStatus::Blocked;
1780 self.state.event(
1781 "setup",
1782 format!("worktree setup failed in {}: {e:#}", wt.display()),
1783 );
1784 self.state.save()?;
1785 return Err(e);
1786 }
1787 Ok(())
1788 }
1789
1790 async fn rollback_prep(&mut self, repo: &Path) {
1792 let root = self.state.worktree_root();
1793 for c in std::mem::take(&mut self.state.candidates) {
1794 if matches!(
1795 git::worktree_remove_clean(repo, &c.worktree).await,
1796 Ok(true)
1797 ) {
1798 git::branch_delete(repo, &c.branch).await.ok();
1799 } else {
1800 self.state.candidates.push(c);
1801 }
1802 }
1803 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1804 for j in 1..=seats {
1805 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1806 .await
1807 .ok();
1808 }
1809 for k in 1..=self.state.config.graph.advisors {
1810 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1811 .await
1812 .ok();
1813 }
1814 }
1815
1816 async fn prep(&mut self) -> Result<()> {
1817 if let Some(reason) = &self.state.prep_failure {
1818 bail!(
1819 "candidate preparation previously failed: {reason}; correct the source before retrying"
1820 );
1821 }
1822 if self
1823 .state
1824 .prep_complete
1825 .unwrap_or(!self.state.candidates.is_empty())
1826 {
1827 return Ok(());
1828 }
1829 let repo = self.state.repo.clone();
1830 let _coordination = self.coordinate(&repo).await?;
1833 let base = self.state.base_commit.clone();
1834 let plan = self.state.sources.plan(&repo, &base).await?;
1835 self.rollback_prep(&repo).await;
1836 if !self.state.candidates.is_empty() {
1837 bail!(
1838 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1839 );
1840 }
1841 self.state.prep_complete = Some(false);
1842 self.state.status = RunStatus::Prep;
1843 self.state.save()?;
1844 let repo = self.state.repo.clone();
1845 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1846 let root = self.state.worktree_root();
1847 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1848
1849 let hooks_dir = self.state.dir().join("hooks");
1852 if self.state.config.blind.commit_msg_hook {
1853 std::fs::create_dir_all(&hooks_dir)
1854 .with_context(|| format!("create {}", hooks_dir.display()))?;
1855 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1856 let path = hooks_dir.join("commit-msg");
1857 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1858 make_executable(&path)?;
1859 git::acquire_worktree_config(&repo).await?;
1867 self.state.enabled_worktree_config = true;
1868 }
1869
1870 ensure_setup_config(&mut self.state, &repo).await?;
1873
1874 for (index, (spec, label)) in self
1875 .roles
1876 .implementers
1877 .clone()
1878 .into_iter()
1879 .zip(labels)
1880 .enumerate()
1881 {
1882 let branch = self.state.branch_for(label);
1883 let worktree = root.join(format!("cand-{label}"));
1884 self.state.event(
1888 "prep-intent",
1889 serde_json::json!({
1890 "branch": branch,
1891 "worktree": worktree,
1892 "start": start,
1893 "picks": plan.picks,
1894 })
1895 .to_string(),
1896 );
1897 self.state.save()?;
1898 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1899 if self.state.config.blind.commit_msg_hook {
1900 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1901 }
1902 git::local_exclude(&worktree, "/.magi/").await?;
1903 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1904 git::worktree_remove_clean(&repo, &worktree).await.ok();
1907 git::branch_delete(&repo, &branch).await.ok();
1908 self.rollback_prep(&repo).await;
1909 self.state.status = RunStatus::Blocked;
1910 self.state
1911 .event("prep", format!("worktree setup failed: {e:#}"));
1912 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1913 self.state.save()?;
1914 return Err(e);
1915 }
1916 self.state.candidates.push(Candidate {
1917 index,
1918 label,
1919 agent: spec.id.clone(),
1920 branch,
1921 worktree,
1922 summary: String::new(),
1923 stat: String::new(),
1924 files: 0,
1925 commits: 0,
1926 empty: false,
1927 failed: None,
1928 verified_noop: None,
1929 duration_ms: 0,
1930 folded: false,
1931 });
1932 self.state.save()?;
1933 for pick in &plan.picks {
1934 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1935 {
1936 self.state.status = RunStatus::Blocked;
1937 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1938 self.state
1939 .event("prep", format!("cannot apply referenced commit: {e}"));
1940 self.state.save()?;
1941 return Err(e);
1942 }
1943 }
1944 }
1945
1946 for j in 1..=self.roles.judges.len() {
1947 let wt = root.join(format!("judge-{j}"));
1948 if !wt.exists() {
1949 git::worktree_add_detached(&repo, &wt, &base).await?;
1950 self.setup_seat_worktree(&repo, &wt, true).await?;
1951 }
1952 }
1953
1954 if self.state.config.graph.advise {
1962 for k in 1..=self.state.config.graph.advisors {
1963 let wt = root.join(format!("advisor-{k}"));
1964 if !wt.exists() {
1965 git::worktree_add_detached(&repo, &wt, &base).await?;
1966 self.setup_seat_worktree(&repo, &wt, true).await?;
1967 }
1968 }
1969 }
1970
1971 let authors: Vec<&str> = self
1976 .roles
1977 .implementers
1978 .iter()
1979 .map(|a| a.id.as_str())
1980 .collect();
1981 let overlap: Vec<String> = self
1982 .roles
1983 .judges
1984 .iter()
1985 .enumerate()
1986 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1987 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1988 .collect();
1989 if !overlap.is_empty() {
1990 let note = format!(
1991 "{} also authored a candidate; blind, but the panel is less \
1992 independent than {} distinct agents would be",
1993 overlap.join(", "),
1994 self.roles.judges.len()
1995 );
1996 self.state.event("prep", note);
1997 }
1998
1999 self.state.event(
2000 "prep",
2001 format!(
2002 "{} candidates, {} judges, base {} ({})",
2003 self.state.candidates.len(),
2004 self.roles.judges.len(),
2005 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2006 self.state.base_branch
2007 ),
2008 );
2009 self.state.prep_complete = Some(true);
2010 self.state.status = RunStatus::Implementing;
2011 self.state.save()?;
2012 Ok(())
2013 }
2014
2015 async fn advise(&mut self) -> Result<()> {
2048 let implement_untouched = self
2049 .state
2050 .candidates
2051 .iter()
2052 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2053 if !self.state.config.graph.advise || self.state.advise_attempted {
2054 return Ok(());
2055 }
2056 if !implement_untouched {
2057 self.state.event(
2058 "advise",
2059 "skipping the design-deliberation stage: at least one \
2060 candidate already shows implementation progress, so this \
2061 run is past the point the stage exists to run before"
2062 .to_owned(),
2063 );
2064 self.state.advise_attempted = true;
2065 self.state.save()?;
2066 return Ok(());
2067 }
2068 let run_id = self.state.id.clone();
2069 let prompts = self.state.config.prompts.clone();
2070 let instruction = self.state.instruction.clone();
2071 let language = self.state.config.graph.language.clone();
2072 let root = self.state.worktree_root();
2073 let n = self.state.config.graph.advisors;
2074 let where_recorded = self.state.dir().join("run.json");
2075
2076 let seats = match self.state.config.advisors() {
2077 Ok(seats) if !seats.is_empty() => seats,
2078 Ok(_) => {
2079 self.state.event(
2080 "advise",
2081 format!(
2082 "[graph] advisors is 0; skipping the design-deliberation \
2083 stage and continuing without a synthesis brief (see {})",
2084 where_recorded.display()
2085 ),
2086 );
2087 self.state.advise_attempted = true;
2088 self.state.save()?;
2089 return Ok(());
2090 }
2091 Err(e) => {
2092 self.state.event(
2093 "advise",
2094 format!(
2095 "could not resolve advisor seats ({e:#}); continuing \
2096 without a design-deliberation brief (see {})",
2097 where_recorded.display()
2098 ),
2099 );
2100 self.state.advise_attempted = true;
2101 self.state.save()?;
2102 return Ok(());
2103 }
2104 };
2105
2106 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2107 let artifacts = agent::artifacts_dir(&self.state.dir());
2108 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2109
2110 let mut jobs = Vec::new();
2111 for (i, spec) in seats.iter().cloned().enumerate() {
2112 let seat_key = format!("advisor-{}", i + 1);
2113 let seat = self.seat(&seat_key, &spec.id);
2114 jobs.push(SeatJob {
2115 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2116 spec,
2117 seat,
2118 cwd: worktrees[i % worktrees.len()].clone(),
2119 timeout,
2120 allow_write: false,
2121 sessions: false,
2122 artifacts: artifacts.clone(),
2123 stem: seat_key,
2124 handover: None,
2125 });
2126 }
2127
2128 self.state.event(
2129 "advise",
2130 format!(
2131 "{} advisor seat(s) sketching a design in parallel",
2132 jobs.len()
2133 ),
2134 );
2135 let mut quota_losses = Vec::new();
2136 let cache = self.state.config.cache_dir();
2137 let ctx = WaveCtx {
2138 carry_seats: false,
2139 pauses: Some((&self.pause, &self.interrupt)),
2140 run: &run_id,
2141 node: "advise",
2142 prompts: &prompts,
2143 cache: cache.as_deref(),
2144 round: None,
2145 };
2146 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2147 let results = ask_json_wave::<Proposal>(
2148 jobs,
2149 Arc::clone(&self.sem),
2150 self.state.config.graph.retries,
2151 &advisor_roster,
2152 &ctx,
2153 &mut quota_losses,
2154 &mut self.state,
2155 &|p: &Proposal| p.validate(),
2156 )
2157 .await;
2158 if self.state.resource_block.is_some() {
2159 return self.stop_for_resource();
2160 }
2161 self.state.quota.extend(quota_losses);
2162
2163 let mut records = Vec::with_capacity(results.len());
2164 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2165 let agent_id = seat.agent.clone();
2166 self.state.seats.insert(seat.key.clone(), seat);
2167 match res {
2168 Ok((proposal, out)) => {
2169 self.state
2170 .event("advise", format!("advisor-{} proposed a design", i + 1));
2171 records.push(advise::AdvisorRecord::proposed(
2172 i + 1,
2173 agent_id,
2174 proposal,
2175 out.duration_ms,
2176 ));
2177 }
2178 Err(e) => {
2179 self.state.event(
2180 "advise",
2181 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2182 );
2183 records.push(advise::AdvisorRecord::failed(
2184 i + 1,
2185 agent_id,
2186 e.to_string(),
2187 ));
2188 }
2189 }
2190 }
2191
2192 let mut advice = advise::Advice {
2193 records,
2194 synthesis: None,
2195 };
2196 if advice.proposals().is_empty() {
2197 self.state.event(
2198 "advise",
2199 "no advisor produced a usable proposal; continuing without a \
2200 synthesis brief"
2201 .to_owned(),
2202 );
2203 } else {
2204 match self
2205 .synthesize_brief(
2206 &advice,
2207 &instruction,
2208 &language,
2209 &worktrees[0],
2210 &artifacts,
2211 &run_id,
2212 &prompts,
2213 cache.as_deref(),
2214 )
2215 .await
2216 {
2217 Ok(Some(text)) => {
2218 self.state.event(
2219 "advise",
2220 "synthesized a design brief for the implementer".to_owned(),
2221 );
2222 advice.synthesis = Some(text);
2223 }
2224 Ok(None) => {
2225 self.state.event(
2226 "advise",
2227 "the synthesis seat produced nothing usable; continuing \
2228 without a design brief"
2229 .to_owned(),
2230 );
2231 }
2232 Err(e) => {
2233 if e.is::<crate::repo_lock::ResourceBlocked>() {
2234 return Err(e);
2235 }
2236 self.state.event(
2237 "advise",
2238 format!("could not synthesize a design brief: {e:#}"),
2239 );
2240 }
2241 }
2242 }
2243 advise::apply_reflection(&mut advice);
2244
2245 self.state.advice = Some(advice);
2246 self.state.advise_attempted = true;
2247 self.state.save()?;
2248 Ok(())
2249 }
2250
2251 #[allow(clippy::too_many_arguments)]
2263 async fn synthesize_brief(
2264 &mut self,
2265 advice: &advise::Advice,
2266 instruction: &str,
2267 language: &str,
2268 cwd: &Path,
2269 artifacts: &Path,
2270 run_id: &str,
2271 prompts: &Prompts,
2272 cache: Option<&Path>,
2273 ) -> Result<Option<String>> {
2274 let chain = agent::pick_chain(
2275 &self.state.config.agents,
2276 self.state.config.roles.synthesizer.as_ref(),
2277 &agent::installed,
2278 "synthesizer",
2279 )?;
2280 let proposals = advice.proposals();
2281 let mut prompt = prompt::with_overlay(
2282 prompt::synthesize_brief(instruction, &proposals, language),
2283 prompts.overlay("advise"),
2284 );
2285 if cache.is_some() {
2286 prompt.push('\n');
2291 prompt.push_str(&prompt::build_cache_note("advise", false));
2292 }
2293 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2294 let mut last = None;
2299 for (n, spec) in chain.iter().enumerate() {
2300 if n > 0 {
2301 self.state
2302 .event("advise", format!("synthesis falling back to {}", spec.id));
2303 }
2304 let mut seat = self.seat("advise-synthesis", &spec.id);
2305 let wait = self.repository_wait();
2306 let outcome = wait
2307 .scope(agent::invoke(
2308 spec,
2309 &mut seat,
2310 &Invocation {
2311 scope: crate::agent::InvocationScope::Repository,
2312 coordination: self.state.config.coordination.as_ref(),
2313 fetched_base: None,
2314 cwd,
2315 prompt: &prompt,
2316 timeout,
2317 allow_write: false,
2318 unsandboxed: false,
2319 sessions: false,
2320 artifacts,
2321 stem: &if n == 0 {
2322 "advise-synthesis".to_owned()
2323 } else {
2324 format!("advise-synthesis-{}", spec.id)
2325 },
2326 run: run_id,
2327 node: "advise",
2328 cache_dir: None,
2329 attachments: &[],
2330 writable: &[],
2331 },
2332 ))
2333 .await;
2334 if outcome.is_ok() {
2335 self.state.seats.insert(seat.key.clone(), seat);
2336 }
2337 let advance = agent::chain_advances(&outcome);
2338 last = Some(outcome);
2339 if !advance {
2340 break;
2341 }
2342 }
2343 let out = last.expect("a chain holds at least one agent")?;
2346 if !out.usable() {
2347 return Ok(None);
2348 }
2349 let text =
2350 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2351 Ok((!text.trim().is_empty()).then_some(text))
2352 }
2353
2354 async fn implement(&mut self) -> Result<()> {
2357 self.persist_candidates().await?;
2358 let run_id = self.state.id.clone();
2363 let prompts = self.state.config.prompts.clone();
2364 let todo: Vec<usize> = self
2365 .state
2366 .candidates
2367 .iter()
2368 .enumerate()
2369 .filter(|(_, c)| {
2370 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2371 && c.failed.is_none()
2372 && !c.empty
2373 })
2374 .map(|(i, _)| i)
2375 .collect();
2376 if todo.is_empty() {
2377 return self.after_implement();
2378 }
2379 self.state.status = RunStatus::Implementing;
2380
2381 let language = self.state.config.graph.language.clone();
2382 let github_language = self.state.config.graph.github_language.clone();
2383 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2384 let sessions = self.state.config.graph.sessions;
2385 let artifacts = agent::artifacts_dir(&self.state.dir());
2386 let brief = self
2390 .state
2391 .advice
2392 .as_ref()
2393 .and_then(|a| a.synthesis.as_deref())
2394 .map(str::to_owned);
2395 let attachments = self.state.attachments.clone();
2396
2397 let mut jobs = Vec::new();
2398 for &i in &todo {
2399 let (index, label, worktree) = {
2400 let c = &self.state.candidates[i];
2401 (c.index, c.label, c.worktree.clone())
2402 };
2403 let spec = self.roles.implementers[index].clone();
2404 let seat_key = format!("impl-{label}");
2405 let seat = self.seat(&seat_key, &spec.id);
2406 let instruction = seeded_instruction(&self.state);
2407 jobs.push(SeatJob {
2408 spec,
2409 seat,
2410 prompt: prompt::implement_in(
2411 &instruction,
2412 &worktree.to_string_lossy(),
2413 &language,
2414 &github_language,
2415 brief.as_deref(),
2416 &attachments,
2417 ),
2418 cwd: worktree,
2419 timeout,
2420 allow_write: true,
2421 sessions,
2422 artifacts: artifacts.clone(),
2423 stem: format!("impl-{label}"),
2424 handover: None,
2425 });
2426 }
2427
2428 self.state.event(
2429 "implement",
2430 format!("{} candidates in parallel", jobs.len()),
2431 );
2432 let mut sent = jobs.clone();
2438 let cache = self.state.config.cache_dir();
2439 let ctx = WaveCtx {
2440 carry_seats: false,
2441 pauses: Some((&self.pause, &self.interrupt)),
2442 run: &run_id,
2443 node: "implement",
2444 prompts: &prompts,
2445 cache: cache.as_deref(),
2446 round: None,
2447 };
2448 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2449 if !wave_blocked(&results) {
2450 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2451 .await;
2452 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2453 .await?;
2454 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2455 .await;
2456 }
2457 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2458
2459 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2460 && results
2461 .iter()
2462 .any(|(_, _, out)| FailClass::of(out).is_some());
2463 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2464 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2465 if self
2469 .state
2470 .candidate_preservation
2471 .contains_key(&self.state.candidates[i].label)
2472 {
2473 self.state.seats.insert(seat.key.clone(), seat);
2474 }
2475 continue;
2476 }
2477 let seat_key = seat.key.clone();
2478 let agent = seat.agent.clone();
2487 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2488 self.state.seats.insert(seat.key.clone(), seat);
2489 let label = self.state.candidates[i].label;
2490 self.state.interrupted_implementations.remove(&label);
2491
2492 let (summary, duration, failed, verified_claim) = match out {
2493 AgentOutcome::Ok(o) => {
2494 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2495 let failed = (!o.usable()).then(|| {
2496 if o.timed_out {
2497 "agent timed out".to_owned()
2498 } else {
2499 format!(
2500 "agent exited with {}",
2501 crate::notices::exit_status(o.exit_code)
2502 )
2503 }
2504 });
2505 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2506 (text, o.duration_ms, failed, verified_claim)
2507 }
2508 AgentOutcome::Dropped(o) => {
2514 let why = o
2515 .dropped
2516 .as_ref()
2517 .map(|d| d.why.as_str())
2518 .unwrap_or("the CLI ended the stream without delivering its answer");
2519 (
2520 String::new(),
2521 o.duration_ms,
2522 Some(format!("the CLI dropped the stream ({why})")),
2523 None,
2524 )
2525 }
2526 AgentOutcome::Quota(o) => {
2527 self.state.quota.push(QuotaLoss {
2528 seat: seat_key,
2529 node: "implement".to_owned(),
2530 at: Timestamp::now(),
2531 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2532 });
2533 (
2534 String::new(),
2535 o.duration_ms,
2536 Some("rate limited (quota); produced no change".to_owned()),
2537 None,
2538 )
2539 }
2540 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2541 (String::new(), 0, Some(e), None)
2542 }
2543 };
2544
2545 let c = &mut self.state.candidates[i];
2548 if !exhausted_the_fallback_chain {
2549 c.agent = agent;
2550 }
2551 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2552 c.duration_ms = duration;
2553 c.failed = failed;
2554 c.verified_noop = verified_claim;
2555 self.state
2556 .candidate_preservation
2557 .insert(label, "awaiting persistence".to_owned());
2558 }
2559 self.state.save()?;
2560 self.persist_candidates().await?;
2561
2562 if resource_blocked {
2563 return self.stop_for_resource();
2564 }
2565 if selected_failed {
2566 self.state.status = RunStatus::Blocked;
2567 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2568 self.state.save()?;
2569 bail!(
2570 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2571 );
2572 }
2573 self.after_implement()
2574 }
2575
2576 async fn persist_candidates(&mut self) -> Result<()> {
2578 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2579 for label in labels {
2580 let i = self
2581 .state
2582 .candidates
2583 .iter()
2584 .position(|c| c.label == label)
2585 .context("pending candidate is missing")?;
2586 let c = self.state.candidates[i].clone();
2587 let interrupted = self.state.interrupted_implementations.contains(&label);
2588 let result = async {
2589 let wait = crate::repo_lock::WaitPolicy::new(
2590 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2591 || false,
2592 );
2593 let _coordination =
2594 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2595 if let Err(e) = git::snapshot_uncommitted(
2598 &c.worktree,
2599 &self
2600 .state
2601 .dir()
2602 .join("artifacts")
2603 .join(format!("candidate-{label}-recovery")),
2604 )
2605 .await
2606 {
2607 self.state.event(
2608 "preserve",
2609 format!("candidate {label} snapshot incomplete: {e:#}"),
2610 );
2611 }
2612 let rescued = git::rescue_commit(
2613 &c.worktree,
2614 &format!("magi: candidate {label} (uncommitted work)"),
2615 )
2616 .await?;
2617 self.state.note_withheld("implement", &rescued.withheld);
2618 let base = &self.state.base_commit;
2619 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2620 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2621 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2622 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2623 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2624 if c.empty && commits > 0 && !patch.trim().is_empty()
2625 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2626 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2627 || self.state.tally.is_some())
2628 {
2629 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2633 }
2634 let c = &mut self.state.candidates[i];
2635 c.commits = commits;
2636 c.stat = stat;
2637 c.files = files;
2638 let no_changes = commits == 0 || patch.trim().is_empty();
2639 c.empty = no_changes && !interrupted;
2640 if interrupted && no_changes {
2641 c.failed = None;
2642 c.verified_noop = None;
2643 }
2644 if !c.empty {
2645 c.failed = None;
2646 c.verified_noop = None;
2647 }
2648 self.state.event(
2649 "preserve",
2650 format!("candidate {label}: {files} files, {commits} commits"),
2651 );
2652 Ok::<_, anyhow::Error>(())
2653 }
2654 .await;
2655 match result {
2656 Ok(()) => {
2657 self.state.candidate_preservation.remove(&label);
2658 if !interrupted || self.state.candidates[i].files > 0 {
2659 self.state.interrupted_implementations.remove(&label);
2660 }
2661 }
2662 Err(e) => {
2663 let reason = format!("candidate {label} preservation failed: {e:#}");
2664 self.state.event("preserve", &reason);
2665 self.state.candidate_preservation.insert(label, reason);
2666 }
2667 }
2668 if self.state.candidate_preservation.is_empty()
2669 && self.state.status == RunStatus::Blocked
2670 && self.state.candidates.iter().all(|c| c.failed.is_none())
2671 {
2672 self.state.status = RunStatus::Implementing;
2676 self.state.event(
2677 "preserve",
2678 "candidate persistence recovered; implementation can continue",
2679 );
2680 }
2681 self.state.save()?;
2682 }
2683 if !self.state.candidate_preservation.is_empty() {
2684 self.state.status = RunStatus::Blocked;
2685 self.state.save()?;
2686 bail!(
2687 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2688 );
2689 }
2690 Ok(())
2691 }
2692
2693 async fn resume_undelivered(
2721 &mut self,
2722 results: &mut [(usize, SeatState, AgentOutcome)],
2723 sent: &[SeatJob],
2724 prompts: &Prompts,
2725 run_id: &str,
2726 ) {
2727 for (wi, seat, out) in results.iter_mut() {
2728 let Some(dropped) = (match &*out {
2729 AgentOutcome::Dropped(o) => o.dropped.clone(),
2730 _ => None,
2731 }) else {
2732 continue;
2733 };
2734 let Some(job) = sent.get(*wi) else { continue };
2735 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2737 self.state.event(
2738 "implement",
2739 format!(
2740 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2741 work is in the tree",
2742 seat.key, dropped.output_tokens, dropped.why
2743 ),
2744 );
2745 continue;
2746 }
2747 if !has_context(&job.spec, seat, job.sessions) {
2755 self.state.event(
2756 "implement",
2757 format!(
2758 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2759 is no session left to resume",
2760 seat.key, dropped.output_tokens, dropped.why
2761 ),
2762 );
2763 continue;
2764 }
2765 self.state.event(
2766 "implement",
2767 format!(
2768 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2769 conversation",
2770 seat.key, dropped.output_tokens, dropped.why
2771 ),
2772 );
2773 let mut retry = job.clone();
2774 retry.seat = seat.clone();
2775 retry.prompt = prompt::resume_after_drop(&dropped.why);
2776 retry.timeout = retry_budget(job.timeout, true);
2777 retry.stem = format!("{}-resume", job.stem);
2778 let cache = self.state.config.cache_dir();
2779 let ctx = WaveCtx {
2780 carry_seats: false,
2781 pauses: Some((&self.pause, &self.interrupt)),
2782 run: run_id,
2783 node: "implement",
2784 prompts,
2785 cache: cache.as_deref(),
2786 round: None,
2787 };
2788 let (resumed_seat, resumed) =
2789 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2790 *seat = resumed_seat;
2791 *out = resumed;
2792 }
2793 }
2794
2795 async fn resume_seat_handovers(
2860 &mut self,
2861 results: &mut [(usize, SeatState, AgentOutcome)],
2862 sent: &mut [SeatJob],
2863 prompts: &Prompts,
2864 run_id: &str,
2865 ) -> Result<()> {
2866 let instruction = seeded_instruction(&self.state);
2867 let language = self.state.config.graph.language.clone();
2868 let github_language = self.state.config.graph.github_language.clone();
2869 let brief = self
2870 .state
2871 .advice
2872 .as_ref()
2873 .and_then(|a| a.synthesis.as_deref())
2874 .map(str::to_owned);
2875 let attachments = self.state.attachments.clone();
2876 for (wi, seat, out) in results.iter_mut() {
2877 let others: BTreeSet<String> = sent
2880 .iter()
2881 .enumerate()
2882 .filter(|(j, _)| j != wi)
2883 .map(|(_, j)| j.spec.id.clone())
2884 .collect();
2885 let Some(job) = sent.get_mut(*wi) else {
2886 continue;
2887 };
2888 let start = self
2893 .roles
2894 .implementer_roster
2895 .iter()
2896 .position(|s| s.id == job.spec.id)
2897 .unwrap_or(0);
2898 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2899 let mut fallback_attempt = 0usize;
2900 let mut prev: Option<FailClass> = None;
2901 while let Some(cur) = FailClass::of(&*out) {
2902 if !should_hand_over(prev.as_ref(), &cur) {
2903 break;
2904 }
2905 let Some(next) =
2906 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2907 .cloned()
2908 else {
2909 break;
2910 };
2911 tried.insert(next.id.clone());
2912 fallback_attempt += 1;
2913
2914 let coordination = self.coordinate(&job.cwd).await?;
2915 if let Ok(r) = git::rescue_commit(
2916 &job.cwd,
2917 &format!(
2918 "magi: candidate {} (uncommitted work before {} fallback)",
2919 seat.key,
2920 if cur == FailClass::Quota {
2921 "quota"
2922 } else {
2923 "handover"
2924 }
2925 ),
2926 )
2927 .await
2928 {
2929 self.state.note_withheld("implement", &r.withheld);
2930 }
2931 drop(coordination);
2932
2933 record_handover(
2934 &mut self.state,
2935 "implement",
2936 &seat.key,
2937 &seat.agent,
2938 &next.id,
2939 &cur,
2940 &fail_reason(&*out),
2941 );
2942 prev = Some(cur.clone());
2943
2944 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2945 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2946 job.spec = next.clone();
2954 let mut retry = job.clone();
2955 retry.seat = new_seat;
2956 retry.prompt = prompt::implement_in(
2957 &instruction,
2958 &job.cwd.to_string_lossy(),
2959 &language,
2960 &github_language,
2961 brief.as_deref(),
2962 &attachments,
2963 );
2964 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2965 let cache = self.state.config.cache_dir();
2966 let ctx = WaveCtx {
2967 carry_seats: false,
2968 pauses: Some((&self.pause, &self.interrupt)),
2969 run: run_id,
2970 node: "implement",
2971 prompts,
2972 cache: cache.as_deref(),
2973 round: None,
2974 };
2975 let (fallback_seat, fallback_out) = run_one(
2976 retry,
2977 Arc::clone(&self.sem),
2978 &ctx,
2979 &mut self.state,
2980 fallback_attempt,
2981 )
2982 .await;
2983 *seat = fallback_seat;
2984 *out = fallback_out;
2985 }
2986 }
2987 Ok(())
2988 }
2989
2990 async fn resume_unconfirmed_commands(
3014 &mut self,
3015 results: &mut [(usize, SeatState, AgentOutcome)],
3016 sent: &[SeatJob],
3017 prompts: &Prompts,
3018 run_id: &str,
3019 ) {
3020 for (wi, seat, out) in results.iter_mut() {
3021 let AgentOutcome::Ok(o) = &*out else {
3022 continue;
3023 };
3024 if !has_unconfirmed_command(&o.commands) {
3025 continue;
3026 }
3027 let Some(job) = sent.get(*wi) else { continue };
3028 if !has_context(&job.spec, seat, job.sessions) {
3029 self.state.event(
3030 "implement",
3031 format!(
3032 "{}: the reply named a command whose own CLI never confirmed the exit \
3033 status of, but there is no session left to resume",
3034 seat.key
3035 ),
3036 );
3037 continue;
3038 }
3039 self.state.event(
3040 "implement",
3041 format!(
3042 "{}: the reply named a command whose own CLI never confirmed the exit \
3043 status of; resuming the conversation",
3044 seat.key
3045 ),
3046 );
3047 let mut retry = job.clone();
3048 retry.seat = seat.clone();
3049 retry.prompt = prompt::resume_incomplete(
3050 "a command in your last reply had no confirmed exit status",
3051 );
3052 retry.timeout = retry_budget(job.timeout, true);
3053 retry.stem = format!("{}-confirm", job.stem);
3054 let cache = self.state.config.cache_dir();
3055 let ctx = WaveCtx {
3056 carry_seats: false,
3057 pauses: Some((&self.pause, &self.interrupt)),
3058 run: run_id,
3059 node: "implement",
3060 prompts,
3061 cache: cache.as_deref(),
3062 round: None,
3063 };
3064 let (resumed_seat, resumed) =
3065 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3066 *seat = resumed_seat;
3067 *out = resumed;
3068 }
3069 }
3070
3071 async fn continue_fix_report(
3092 &mut self,
3093 mut seat: SeatState,
3094 parse_err: String,
3095 job: &SeatJob,
3096 prompts: &Prompts,
3097 run_id: &str,
3098 round: usize,
3099 ) -> (
3100 SeatState,
3101 Option<FixReport>,
3102 Option<String>,
3103 ContinuationRecord,
3104 ) {
3105 let mut last_err = parse_err;
3106 let mut cumulative_wait_ms = 0u64;
3107 let mut attempts = 0usize;
3108 loop {
3109 if !has_context(&job.spec, &seat, job.sessions) {
3110 self.state.event(
3111 "fix",
3112 format!(
3113 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3114 session left to resume into"
3115 ),
3116 );
3117 let outcome = if attempts == 0 {
3118 ContinuationOutcome::NoSession
3119 } else {
3120 ContinuationOutcome::Exhausted
3121 };
3122 return (
3123 seat,
3124 None,
3125 Some(format!("unparsable fix report: {last_err}")),
3126 ContinuationRecord {
3127 attempts,
3128 cumulative_wait_ms,
3129 outcome,
3130 },
3131 );
3132 }
3133 if attempts >= MAX_FIX_CONTINUATIONS {
3134 self.state.event(
3135 "fix",
3136 format!(
3137 "round {round}: fixer's reply still had no adoption report after \
3138 {attempts} continuation(s) ({last_err}); giving up"
3139 ),
3140 );
3141 return (
3142 seat,
3143 None,
3144 Some(format!(
3145 "unparsable fix report after {attempts} continuation(s): {last_err}"
3146 )),
3147 ContinuationRecord {
3148 attempts,
3149 cumulative_wait_ms,
3150 outcome: ContinuationOutcome::Exhausted,
3151 },
3152 );
3153 }
3154 attempts += 1;
3155 self.state.event(
3156 "fix",
3157 format!(
3158 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3159 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3160 ),
3161 );
3162 let mut retry = job.clone();
3163 retry.seat = seat.clone();
3164 retry.prompt = prompt::resume_incomplete(&last_err);
3165 retry.timeout = retry_budget(job.timeout, true);
3166 retry.stem = format!("{}-continue{attempts}", job.stem);
3167 let cache = self.state.config.cache_dir();
3168 let ctx = WaveCtx {
3169 carry_seats: false,
3170 pauses: Some((&self.pause, &self.interrupt)),
3171 run: run_id,
3172 node: "fix",
3173 prompts,
3174 cache: cache.as_deref(),
3175 round: Some(round),
3176 };
3177 let (resumed_seat, resumed_out) = run_one(
3178 retry,
3179 Arc::clone(&self.sem),
3180 &ctx,
3181 &mut self.state,
3182 attempts,
3183 )
3184 .await;
3185 seat = resumed_seat;
3186 match resumed_out {
3187 AgentOutcome::Ok(o) => {
3188 cumulative_wait_ms += o.duration_ms;
3189 match verdict::extract_json::<FixReport>(&o.text) {
3190 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3191 self.state.event(
3192 "fix",
3193 format!(
3194 "round {round}: fixer's adoption report recovered after \
3195 {attempts} continuation(s)"
3196 ),
3197 );
3198 return (
3199 seat,
3200 Some(report),
3201 None,
3202 ContinuationRecord {
3203 attempts,
3204 cumulative_wait_ms,
3205 outcome: ContinuationOutcome::Resumed,
3206 },
3207 );
3208 }
3209 Ok(_) => {
3217 last_err = "the reply parsed, but it reported a command whose own CLI \
3218 never confirmed an exit status"
3219 .to_owned();
3220 }
3221 Err(e) => last_err = e.to_string(),
3222 }
3223 }
3224 AgentOutcome::Quota(o) => {
3225 cumulative_wait_ms += o.duration_ms;
3226 self.state.quota.push(QuotaLoss {
3227 seat: seat.key.clone(),
3228 node: "fix".to_owned(),
3229 at: Timestamp::now(),
3230 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3231 });
3232 self.state.event(
3233 "fix",
3234 format!(
3235 "round {round}: continuation rate limited (quota); not retrying now"
3236 ),
3237 );
3238 return (
3239 seat,
3240 None,
3241 Some("rate limited (quota) while recovering the fix report".to_owned()),
3242 ContinuationRecord {
3243 attempts,
3244 cumulative_wait_ms,
3245 outcome: ContinuationOutcome::QuotaLost,
3246 },
3247 );
3248 }
3249 AgentOutcome::Dropped(o) => {
3250 cumulative_wait_ms += o.duration_ms;
3251 let why = o
3252 .dropped
3253 .as_ref()
3254 .map(|d| d.why.as_str())
3255 .unwrap_or("the CLI ended the stream without delivering its answer");
3256 last_err = format!("the CLI dropped the stream ({why})");
3257 }
3258 AgentOutcome::Failed(e) => last_err = e,
3259 AgentOutcome::ResourceBlocked(e) => {
3262 return (
3263 seat,
3264 None,
3265 Some(format!("the continuation was not started: {e}")),
3266 ContinuationRecord {
3267 attempts: attempts.saturating_sub(1),
3268 cumulative_wait_ms,
3269 outcome: ContinuationOutcome::NoSession,
3270 },
3271 );
3272 }
3273 }
3274 }
3275 }
3276
3277 fn after_implement(&mut self) -> Result<()> {
3278 if !self.state.candidate_preservation.is_empty() {
3279 bail!("candidate preservation incomplete; judging is unsafe");
3280 }
3281 if self.state.leaks.is_empty() {
3283 let cfg = self.state.config.blind.clone();
3284 let mut leaks = Vec::new();
3285 for c in &self.state.candidates {
3286 let Some(patch) =
3287 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3288 else {
3289 continue;
3290 };
3291 leaks.extend(blind::scan(
3292 &format!("candidate {} patch", c.label),
3293 &patch,
3294 &cfg.vendor_tokens,
3295 ));
3296 }
3297 if !leaks.is_empty() {
3298 let summary = leaks
3299 .iter()
3300 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3301 .collect::<Vec<_>>()
3302 .join(", ");
3303 match cfg.on_leak {
3304 LeakPolicy::Fail => {
3305 self.state.status = RunStatus::Failed;
3306 self.state
3307 .event("blind", format!("vendor text in a patch: {summary}"));
3308 self.state.leaks = leaks;
3309 self.state.save()?;
3310 self.settle_questions();
3311 bail!(
3312 "blind.on_leak = \"fail\" and vendor text reached a \
3313 judged patch: {summary}"
3314 );
3315 }
3316 LeakPolicy::Redact => self.state.event(
3317 "blind",
3318 format!("redacting vendor text for judging: {summary}"),
3319 ),
3320 LeakPolicy::Warn => self.state.event(
3321 "blind",
3322 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3323 ),
3324 }
3325 self.state.leaks = leaks;
3326 }
3327 }
3328
3329 if self.state.viable().is_empty() {
3330 if self.state.all_candidates_verified_noop() {
3331 self.state.status = RunStatus::VerifiedNoop;
3342 self.state.save()?;
3343 self.settle_questions();
3344 return Ok(());
3345 }
3346 self.state.status = RunStatus::Failed;
3347 self.state.save()?;
3348 self.settle_questions();
3349 bail!("no candidate produced a change; nothing to judge");
3350 }
3351 self.state.status = RunStatus::Judging;
3352 self.state.save()?;
3353 Ok(())
3354 }
3355
3356 async fn judge(&mut self) -> Result<()> {
3359 let run_id = self.state.id.clone();
3364 let prompts = self.state.config.prompts.clone();
3365 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3366 return Ok(());
3367 }
3368 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3369 if viable.len() == 1 {
3370 self.state.judge_skipped = true;
3377 self.state.event(
3378 "judge",
3379 format!(
3380 "only candidate {} produced a change; judging skipped",
3381 viable[0].label
3382 ),
3383 );
3384 self.state.save()?;
3385 return Ok(());
3386 }
3387 self.state.status = RunStatus::Judging;
3388
3389 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3390 let language = self.state.config.graph.language.clone();
3391 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3392 let sessions = self.state.config.graph.sessions;
3393 let artifacts = agent::artifacts_dir(&self.state.dir());
3394 let root = self.state.worktree_root();
3395 let base_short = short(&self.state.base_commit);
3396
3397 let mut jobs = Vec::new();
3398 let mut orders = Vec::new();
3399 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3400 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3401 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3402 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3403 let seat_key = format!("judge-{}", j + 1);
3404 let seat = self.seat(&seat_key, &spec.id);
3405 jobs.push(SeatJob {
3406 prompt: prompt::judge(
3407 &self.state.instruction,
3408 &views,
3409 self.roles.judges.len(),
3410 &base_short,
3411 &language,
3412 ),
3413 spec,
3414 seat,
3415 cwd: root.join(format!("judge-{}", j + 1)),
3416 timeout,
3417 allow_write: false,
3418 sessions,
3419 artifacts: artifacts.clone(),
3420 stem: format!("judge-{}", j + 1),
3421 handover: None,
3422 });
3423 }
3424
3425 self.state.event(
3426 "judge",
3427 format!(
3428 "{} judges ranking {} candidates blind",
3429 jobs.len(),
3430 viable.len()
3431 ),
3432 );
3433 let labels_for_check = labels.clone();
3434 let mut quota_losses = Vec::new();
3435 let cache = self.state.config.cache_dir();
3436 let ctx = WaveCtx {
3437 carry_seats: false,
3438 pauses: Some((&self.pause, &self.interrupt)),
3439 run: &run_id,
3440 node: "judge",
3441 prompts: &prompts,
3442 cache: cache.as_deref(),
3443 round: None,
3444 };
3445 let results = ask_json_wave::<Ranking>(
3446 jobs,
3447 Arc::clone(&self.sem),
3448 self.state.config.graph.retries,
3449 &self.roles.judge_roster,
3450 &ctx,
3451 &mut quota_losses,
3452 &mut self.state,
3453 &move |r: &Ranking| r.validate(&labels_for_check),
3454 )
3455 .await;
3456 if self.state.resource_block.is_some() {
3457 return self.stop_for_resource();
3458 }
3459 self.state.quota.extend(quota_losses);
3460
3461 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3462 let agent_id = seat.agent.clone();
3463 self.state.seats.insert(seat.key.clone(), seat);
3464 let mut record = Judgement {
3465 judge: j + 1,
3466 seat: format!("judge-{}", j + 1),
3467 agent: agent_id,
3468 ranking: Vec::new(),
3469 reasons: BTreeMap::new(),
3470 confidence: None,
3471 strengths: BTreeMap::new(),
3472 order: orders[j].clone(),
3473 failed: None,
3474 duration_ms: 0,
3475 };
3476 match res {
3477 Ok((ranking, out)) => {
3478 record.ranking = ranking.normalized();
3479 record.reasons = ranking.reasons;
3480 record.strengths = ranking.strengths;
3481 record.confidence = ranking.confidence;
3482 record.duration_ms = out.duration_ms;
3483 self.state.event(
3484 "judge",
3485 format!(
3486 "judge {} ranked {}",
3487 j + 1,
3488 record.ranking.iter().collect::<String>()
3489 ),
3490 );
3491 }
3492 Err(e) => {
3493 record.failed = Some(e.to_string());
3494 self.state
3495 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3496 }
3497 }
3498 self.state.judgements.push(record);
3499 self.state.save()?;
3500 }
3501 Ok(())
3502 }
3503
3504 async fn deliberate(&mut self) -> Result<()> {
3507 let run_id = self.state.id.clone();
3512 let prompts = self.state.config.prompts.clone();
3513 if !self.state.deliberation.is_empty() {
3514 return Ok(());
3515 }
3516 let tops: Vec<char> = self
3517 .state
3518 .judgements
3519 .iter()
3520 .filter_map(|j| j.ranking.first().copied())
3521 .collect();
3522 let rounds = self.state.config.graph.deliberate_rounds;
3523 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3524 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3525 self.state.event(
3526 "deliberate",
3527 format!("judges agreed on {} outright; no deliberation", tops[0]),
3528 );
3529 }
3530 self.state.status = RunStatus::Voting;
3531 self.state.save()?;
3532 return Ok(());
3533 }
3534
3535 self.state.status = RunStatus::Deliberating;
3536 self.state.event(
3537 "deliberate",
3538 format!(
3539 "split: first choices were {} — opening {rounds} round(s)",
3540 tops.iter().collect::<String>()
3541 ),
3542 );
3543
3544 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3545 let language = self.state.config.graph.language.clone();
3546 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3547 let sessions = self.state.config.graph.sessions;
3548 let artifacts = agent::artifacts_dir(&self.state.dir());
3549 let root = self.state.worktree_root();
3550 let base_short = short(&self.state.base_commit);
3551
3552 for round in 1..=rounds {
3556 let mut turns: Vec<DeliberationTurn> = Vec::new();
3557 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3558 if self.state.judgements[j].failed.is_some() {
3559 continue;
3560 }
3561 let seat_key = format!("judge-{}", j + 1);
3562 let spec = self.occupant(&seat_key, spec);
3563 let mut seat = self.seat(&seat_key, &spec.id);
3564 let transcript = self.transcript(&turns, j);
3565 let build = |context: Option<&str>| {
3566 prompt::deliberate(
3567 &self.state.instruction,
3568 context,
3569 &transcript,
3570 round,
3571 rounds,
3572 &language,
3573 )
3574 };
3575 let block = self.candidate_block(&viable, &base_short);
3576 let full = build(Some(&block));
3577 let text = if has_context(&spec, &seat, sessions) {
3578 build(None)
3579 } else {
3580 full.clone()
3581 };
3582 let job = SeatJob {
3583 spec,
3584 seat: seat.clone(),
3585 prompt: text,
3586 cwd: root.join(format!("judge-{}", j + 1)),
3587 timeout,
3588 allow_write: false,
3589 sessions,
3590 artifacts: artifacts.clone(),
3591 stem: format!("delib-{round}-judge-{}", j + 1),
3592 handover: Some(full),
3593 };
3594 let cache = self.state.config.cache_dir();
3595 let ctx = WaveCtx {
3596 carry_seats: false,
3597 pauses: Some((&self.pause, &self.interrupt)),
3598 run: &run_id,
3599 node: "deliberate",
3600 prompts: &prompts,
3601 cache: cache.as_deref(),
3602 round: None,
3603 };
3604 let mut losses = Vec::new();
3608 let mut results = ask_wave_with::<String>(
3609 vec![job],
3610 Arc::clone(&self.sem),
3611 0,
3612 &self.roles.judge_roster,
3613 &ctx,
3614 &mut losses,
3615 &mut self.state,
3616 &|text: &str| {
3617 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3618 },
3619 )
3620 .await;
3621 self.state.quota.extend(losses);
3622 let (updated, res, _) = results.pop().expect("one job in, one result out");
3623 seat = updated;
3624 let agent_id = seat.agent.clone();
3625 self.state.seats.insert(seat.key.clone(), seat);
3626 let body = match res {
3627 Ok((body, _)) => body,
3628 Err(e) => {
3631 self.state
3632 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3633 continue;
3634 }
3635 };
3636 let tentative = verdict::extract_json::<Position>(&body)
3637 .ok()
3638 .and_then(|p| p.tentative)
3639 .and_then(|s| s.trim().chars().next())
3640 .map(|c| c.to_ascii_uppercase());
3641 self.state.event(
3642 "deliberate",
3643 format!(
3644 "round {round}: judge {} now favours {}",
3645 j + 1,
3646 tentative.map_or("—".to_owned(), |c| c.to_string())
3647 ),
3648 );
3649 turns.push(DeliberationTurn {
3650 judge: j + 1,
3651 agent: agent_id,
3652 body: blind::sanitize_prose(&body, &self.state.config.blind),
3653 tentative,
3654 });
3655 }
3656 self.state
3657 .deliberation
3658 .push(DeliberationRound { round, turns });
3659 self.state.save()?;
3660 }
3661
3662 self.state.status = RunStatus::Voting;
3663 self.state.save()?;
3664 Ok(())
3665 }
3666
3667 async fn vote(&mut self) -> Result<()> {
3670 let run_id = self.state.id.clone();
3675 let prompts = self.state.config.prompts.clone();
3676 if !self.state.votes.is_empty() {
3677 return Ok(());
3678 }
3679 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3680 if viable.len() == 1 {
3681 return Ok(());
3682 }
3683 self.state.status = RunStatus::Voting;
3684
3685 let language = self.state.config.graph.language.clone();
3686 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3687 let sessions = self.state.config.graph.sessions;
3688 let artifacts = agent::artifacts_dir(&self.state.dir());
3689 let root = self.state.worktree_root();
3690 let base_short = short(&self.state.base_commit);
3691 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3692
3693 let mut jobs = Vec::new();
3694 let mut seats_at = Vec::new();
3695 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3696 if self
3697 .state
3698 .judgements
3699 .get(j)
3700 .is_some_and(|r| r.failed.is_some())
3701 {
3702 continue;
3703 }
3704 let seat_key = format!("judge-{}", j + 1);
3705 let spec = self.occupant(&seat_key, spec);
3706 let seat = self.seat(&seat_key, &spec.id);
3707 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3708 let text = if has_context(&spec, &seat, sessions) {
3709 prompt::final_vote(&viable, &language)
3710 } else {
3711 full.clone()
3712 };
3713 jobs.push(SeatJob {
3714 spec,
3715 seat,
3716 prompt: text,
3717 cwd: root.join(format!("judge-{}", j + 1)),
3718 timeout,
3719 allow_write: false,
3720 sessions,
3721 artifacts: artifacts.clone(),
3722 stem: format!("vote-judge-{}", j + 1),
3723 handover: Some(full),
3724 });
3725 seats_at.push(j);
3726 }
3727
3728 self.state.event(
3729 "vote",
3730 format!(
3731 "collecting {} final votes one by one, privately",
3732 jobs.len()
3733 ),
3734 );
3735 let allowed = viable.clone();
3736 let mut quota_losses = Vec::new();
3737 let cache = self.state.config.cache_dir();
3738 let ctx = WaveCtx {
3739 carry_seats: false,
3740 pauses: Some((&self.pause, &self.interrupt)),
3741 run: &run_id,
3742 node: "vote",
3743 prompts: &prompts,
3744 cache: cache.as_deref(),
3745 round: None,
3746 };
3747 let results = ask_json_wave::<FinalVote>(
3748 jobs,
3749 Arc::clone(&self.sem),
3750 self.state.config.graph.retries,
3751 &self.roles.judge_roster,
3752 &ctx,
3753 &mut quota_losses,
3754 &mut self.state,
3755 &move |v: &FinalVote| match v.label() {
3756 Some(c) if allowed.contains(&c) => Ok(()),
3757 other => bail!("vote {other:?} is not one of {allowed:?}"),
3758 },
3759 )
3760 .await;
3761 if self.state.resource_block.is_some() {
3762 return self.stop_for_resource();
3763 }
3764 self.state.quota.extend(quota_losses);
3765
3766 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3767 let agent_id = seat.agent.clone();
3768 self.state.seats.insert(seat.key.clone(), seat);
3769 let initial = self
3770 .state
3771 .judgements
3772 .get(j)
3773 .and_then(|r| r.ranking.first().copied());
3774 let mut record = VoteRecord {
3775 judge: j + 1,
3776 agent: agent_id,
3777 vote: None,
3778 reason: String::new(),
3779 changed: false,
3780 };
3781 match res {
3782 Ok((v, _)) => {
3783 record.vote = v.label();
3784 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3785 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3786 self.state.event(
3787 "vote",
3788 format!(
3789 "judge {} voted {}{}",
3790 j + 1,
3791 record.vote.unwrap_or('?'),
3792 if record.changed { " (changed)" } else { "" }
3793 ),
3794 );
3795 }
3796 Err(e) => {
3797 self.state
3798 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3799 }
3800 }
3801 self.state.votes.push(record);
3802 self.state.save()?;
3803 }
3804 Ok(())
3805 }
3806
3807 fn tally(&mut self) -> Result<()> {
3810 if self.state.tally.is_some() {
3811 return Ok(());
3812 }
3813 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3814 let tops: Vec<char> = self
3815 .state
3816 .judgements
3817 .iter()
3818 .filter_map(|j| j.ranking.first().copied())
3819 .collect();
3820 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3821
3822 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3825 let mut cast: Vec<char> = Vec::new();
3826 for (i, j) in self.state.judgements.iter().enumerate() {
3827 let vote = self
3828 .state
3829 .votes
3830 .iter()
3831 .find(|v| v.judge == i + 1)
3832 .and_then(|v| v.vote)
3833 .or_else(|| j.ranking.first().copied());
3834 if let Some(v) = vote {
3835 *first_choice.entry(v).or_insert(0) += 1;
3836 cast.push(v);
3837 }
3838 }
3839
3840 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3841 for j in &self.state.judgements {
3842 let n = j.ranking.len();
3843 for (pos, label) in j.ranking.iter().enumerate() {
3844 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3845 }
3846 }
3847
3848 let best = first_choice.values().copied().max().unwrap_or(0);
3849 let mut leaders: Vec<char> = first_choice
3850 .iter()
3851 .filter(|(_, v)| **v == best)
3852 .map(|(k, _)| *k)
3853 .collect();
3854 let mut tie_break = None;
3855 if leaders.len() > 1 {
3856 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3857 let borda_leaders: Vec<char> = leaders
3858 .iter()
3859 .copied()
3860 .filter(|l| borda[l] == top_borda)
3861 .collect();
3862 tie_break = Some(if borda_leaders.len() == 1 {
3863 format!(
3864 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3865 leaders.len()
3866 )
3867 } else {
3868 format!(
3869 "{} way tie on both first-choice votes and Borda points, broken by label order",
3870 leaders.len()
3871 )
3872 });
3873 leaders = borda_leaders;
3874 leaders.sort_unstable();
3875 }
3876 let winner = *leaders
3877 .first()
3878 .or(viable.first())
3879 .context("no candidate to declare a winner from")?;
3880
3881 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3882 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3883 let deliberated = !self.state.deliberation.is_empty();
3884
3885 let quota_seats: std::collections::BTreeSet<&str> =
3889 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3890 let mut present = 0usize;
3891 for (i, j) in self.state.judgements.iter().enumerate() {
3892 if quota_seats.contains(j.seat.as_str()) {
3893 continue;
3894 }
3895 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3896 let voted = self
3897 .state
3898 .votes
3899 .iter()
3900 .any(|v| v.judge == i + 1 && v.vote.is_some());
3901 if ranked || voted {
3902 present += 1;
3903 }
3904 }
3905 let needs_quorum = viable.len() > 1;
3911 let judges_total = if needs_quorum {
3912 self.roles.judges.len()
3913 } else {
3914 0
3915 };
3916 let quorum = if needs_quorum {
3917 judges_total / 2 + 1
3918 } else {
3919 0
3920 };
3921 let met_quorum = !needs_quorum || present >= quorum;
3922 let uncontested = (!needs_quorum).then(|| {
3923 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3924 });
3925
3926 self.state.event(
3927 "tally",
3928 match &uncontested {
3929 Some(reason) => format!("winner {winner} — {reason}"),
3930 None => format!(
3931 "winner {winner} — votes {} | initial {} | {} changed | \
3932 {present}/{judges_total} judges{}",
3933 first_choice
3934 .iter()
3935 .map(|(k, v)| format!("{k}:{v}"))
3936 .collect::<Vec<_>>()
3937 .join(" "),
3938 if unanimous_initial {
3939 "unanimous"
3940 } else {
3941 "split"
3942 },
3943 changed_votes,
3944 if met_quorum {
3945 String::new()
3946 } else {
3947 format!(" — below quorum ({quorum} required)")
3948 },
3949 ),
3950 },
3951 );
3952 if !met_quorum {
3953 self.state.event(
3954 "stall",
3955 format!(
3956 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3957 the run stops here, resumable"
3958 ),
3959 );
3960 }
3961 self.state.tally = Some(Tally {
3962 first_choice,
3963 borda,
3964 winner,
3965 rankings: tops.len(),
3966 unanimous_initial,
3967 deliberated,
3968 changed_votes,
3969 unanimous_final,
3970 tie_break,
3971 judges: judges_total,
3972 present,
3973 quorum,
3974 met_quorum,
3975 uncontested,
3976 });
3977 self.state.status = if met_quorum {
3978 RunStatus::Reviewing
3979 } else {
3980 RunStatus::Stalled
3981 };
3982 self.state.save()?;
3983 Ok(())
3984 }
3985
3986 #[allow(clippy::too_many_lines)]
4007 async fn recover_stall(&mut self) -> Result<bool> {
4008 let run_id = self.state.id.clone();
4013 let prompts = self.state.config.prompts.clone();
4014 let quota_seats: BTreeSet<&str> =
4019 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4020 let absent: Vec<String> = self
4021 .state
4022 .judgements
4023 .iter()
4024 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4025 .map(|j| j.seat.clone())
4026 .collect();
4027 if absent.is_empty() {
4028 return Ok(false);
4029 }
4030 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4031 if viable.len() <= 1 {
4032 return Ok(false);
4033 }
4034 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4035 let language = self.state.config.graph.language.clone();
4036 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4037 let sessions = self.state.config.graph.sessions;
4038 let artifacts = agent::artifacts_dir(&self.state.dir());
4039 let root = self.state.worktree_root();
4040 let base_short = short(&self.state.base_commit);
4041 let candidates: Vec<Candidate> = viable.clone();
4042
4043 let mut positions: Vec<usize> = absent
4045 .iter()
4046 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4047 .collect();
4048 if positions.is_empty() {
4049 return Ok(false);
4050 }
4051 positions.sort_unstable();
4052 positions.dedup();
4053
4054 let mut judge_jobs = Vec::new();
4056 for &j in &positions {
4057 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4058 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4059 let seat_key = format!("judge-{}", j + 1);
4060 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4061 let seat = self.seat(&seat_key, &spec.id);
4062 judge_jobs.push(SeatJob {
4063 spec,
4064 seat,
4065 prompt: prompt::judge(
4066 &self.state.instruction,
4067 &views,
4068 self.roles.judges.len(),
4069 &base_short,
4070 &language,
4071 ),
4072 cwd: root.join(seat_key),
4073 timeout,
4074 allow_write: false,
4075 sessions,
4076 artifacts: artifacts.clone(),
4077 stem: format!("judge-{}-recover", j + 1),
4078 handover: None,
4079 });
4080 }
4081
4082 let labels_for_check = labels.clone();
4083 let mut judge_losses = Vec::new();
4084 let retries = self.state.config.graph.retries;
4085 let cache = self.state.config.cache_dir();
4086 let ctx = WaveCtx {
4087 carry_seats: false,
4088 pauses: Some((&self.pause, &self.interrupt)),
4089 run: &run_id,
4090 node: "judge",
4091 prompts: &prompts,
4092 cache: cache.as_deref(),
4093 round: None,
4094 };
4095 let results = ask_json_wave::<Ranking>(
4096 judge_jobs,
4097 Arc::clone(&self.sem),
4098 retries,
4099 &self.roles.judge_roster,
4100 &ctx,
4101 &mut judge_losses,
4102 &mut self.state,
4103 &move |r: &Ranking| r.validate(&labels_for_check),
4104 )
4105 .await;
4106 if self.state.resource_block.is_some() {
4107 self.stop_for_resource()?;
4108 return Ok(false);
4109 }
4110
4111 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4113 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4114 let agent_id = seat.agent.clone();
4115 self.state.seats.insert(seat.key.clone(), seat);
4116 let record = &mut self.state.judgements[j];
4117 match res {
4118 Ok((ranking, out)) => {
4119 record.agent = agent_id;
4120 record.ranking = ranking.normalized();
4121 record.reasons = ranking.reasons;
4122 record.strengths = ranking.strengths;
4123 record.confidence = ranking.confidence;
4124 record.failed = None;
4125 record.duration_ms = out.duration_ms;
4126 recovered.insert(j);
4127 self.state.event(
4128 "recover",
4129 format!("judge {} ranked again after the limit", j + 1),
4130 );
4131 }
4132 Err(e) => {
4133 self.state
4134 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4135 }
4136 }
4137 }
4138
4139 let mut vote_jobs = Vec::new();
4141 let mut vote_pos: Vec<usize> = Vec::new();
4142 for &j in &recovered {
4143 let seat_key = format!("judge-{}", j + 1);
4144 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4145 let seat = self.seat(&seat_key, &spec.id);
4146 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4147 let text = if has_context(&spec, &seat, sessions) {
4148 prompt::final_vote(&labels, &language)
4149 } else {
4150 full.clone()
4151 };
4152 vote_jobs.push(SeatJob {
4153 spec,
4154 seat,
4155 prompt: text,
4156 cwd: root.join(seat_key),
4157 timeout,
4158 allow_write: false,
4159 sessions,
4160 artifacts: artifacts.clone(),
4161 stem: format!("vote-judge-{}-recover", j + 1),
4162 handover: Some(full),
4163 });
4164 vote_pos.push(j);
4165 }
4166 let allowed = labels.clone();
4167 let mut vote_losses = Vec::new();
4168 let vote_retries = self.state.config.graph.retries;
4169 let vote_cache = self.state.config.cache_dir();
4170 let ctx = WaveCtx {
4171 carry_seats: false,
4172 pauses: Some((&self.pause, &self.interrupt)),
4173 run: &run_id,
4174 node: "vote",
4175 prompts: &prompts,
4176 cache: vote_cache.as_deref(),
4177 round: None,
4178 };
4179 let votes = ask_json_wave::<FinalVote>(
4180 vote_jobs,
4181 Arc::clone(&self.sem),
4182 vote_retries,
4183 &self.roles.judge_roster,
4184 &ctx,
4185 &mut vote_losses,
4186 &mut self.state,
4187 &move |v: &FinalVote| match v.label() {
4188 Some(c) if allowed.contains(&c) => Ok(()),
4189 other => bail!("vote {other:?} is not one of {allowed:?}"),
4190 },
4191 )
4192 .await;
4193 if self.state.resource_block.is_some() {
4194 self.stop_for_resource()?;
4195 return Ok(false);
4196 }
4197 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4198 let agent_id = seat.agent.clone();
4199 self.state.seats.insert(seat.key.clone(), seat);
4200 match res {
4201 Ok((v, _)) => {
4202 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4203 rec.vote = v.label();
4204 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4205 } else {
4206 self.state.votes.push(VoteRecord {
4207 judge: j + 1,
4208 agent: agent_id,
4209 vote: v.label(),
4210 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4211 changed: false,
4212 });
4213 }
4214 self.state.event(
4215 "recover",
4216 format!("judge {} voted again after the limit", j + 1),
4217 );
4218 }
4219 Err(e) => {
4220 self.state
4221 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4222 }
4223 }
4224 }
4225
4226 let recovered_keys: BTreeSet<String> = recovered
4230 .iter()
4231 .map(|&j| format!("judge-{}", j + 1))
4232 .collect();
4233 self.state
4234 .quota
4235 .retain(|q| !recovered_keys.contains(&q.seat));
4236 for loss in judge_losses.into_iter().chain(vote_losses) {
4240 if recovered_keys.contains(&loss.seat) {
4241 continue;
4242 }
4243 self.state.quota.retain(|q| q.seat != loss.seat);
4244 self.state.quota.push(loss);
4245 }
4246
4247 self.state.tally = None;
4249 self.tally()?;
4250 Ok(self
4251 .state
4252 .tally
4253 .as_ref()
4254 .map(|t| t.met_quorum)
4255 .unwrap_or(false))
4256 }
4257
4258 async fn graft(&mut self) -> Result<()> {
4274 let Err(e) = self.graft_inner().await else {
4275 return Ok(());
4276 };
4277 if e.is::<crate::repo_lock::ResourceBlocked>() {
4278 return Err(e);
4279 }
4280 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4281 self.state.status = RunStatus::Blocked;
4282 self.state.event("graft", format!("blocked: {e:#}"));
4283 self.state.save()?;
4284 return Err(e);
4285 }
4286 if let Some(rec) = self.state.graft.clone()
4291 && !rec.outcome.settled()
4292 {
4293 if let Some(winner) = self.state.winner().cloned() {
4294 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4295 if re.is::<crate::repo_lock::ResourceBlocked>() {
4296 return Err(re);
4297 }
4298 self.state.status = RunStatus::Blocked;
4299 self.state.event("graft", format!("blocked: {re:#}"));
4300 self.state.save()?;
4301 return Err(re);
4302 }
4303 }
4304 if let Some(g) = self.state.graft.as_mut() {
4305 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4306 }
4307 }
4308 self.state.event("graft", format!("skipped: {e:#}"));
4309 if let Err(e) = self.state.save() {
4310 tracing::warn!("save after graft: {e:#}");
4311 }
4312 Ok(())
4313 }
4314
4315 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4318 let _coordination = self.coordinate(wt).await?;
4319 crate::recovery::restore_repair(wt, pre)
4320 .await
4321 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4322 }
4323
4324 fn graft_skipped(&mut self, why: &str) {
4326 let message = format!("skipped: {why}");
4327 if !self
4328 .state
4329 .events
4330 .iter()
4331 .any(|e| e.node == "graft" && e.message == message)
4332 {
4333 self.state.event("graft", message);
4334 }
4335 }
4336
4337 async fn graft_inner(&mut self) -> Result<()> {
4338 if let Some(rec) = self.state.graft.clone()
4341 && !rec.outcome.settled()
4342 {
4343 if self.state.reviews.is_empty()
4347 && let Some(winner) = self.state.winner().cloned()
4348 {
4349 self.graft_restore(&winner.worktree, &rec.pre).await?;
4350 }
4351 if rec.outcome == GraftOutcome::WaitingForRestore {
4352 self.state.graft = None;
4353 self.state.save()?;
4354 } else {
4355 if let Some(g) = self.state.graft.as_mut() {
4356 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4357 }
4358 self.state.event(
4359 "graft",
4360 "rolled back: the previous round was interrupted; the winner is untouched",
4361 );
4362 self.state.save()?;
4363 return Ok(());
4364 }
4365 }
4366 let plan = match graft::plan(&self.state) {
4367 Ok(plan) => plan,
4368 Err(skip) => {
4369 if skip != graft::Skip::AlreadyDone {
4370 self.graft_skipped(skip.reason());
4371 self.state.save()?;
4372 }
4373 return Ok(());
4374 }
4375 };
4376 let Some(winner) = self.state.winner().cloned() else {
4377 return Ok(());
4378 };
4379 let wt = winner.worktree.clone();
4380 if !git::is_clean(&wt).await? {
4381 self.graft_skipped("the winner's worktree has uncommitted changes");
4382 self.state.save()?;
4383 return Ok(());
4384 }
4385 let pre = git::rev_parse(&wt, "HEAD").await?;
4386 self.state.graft = Some(GraftRecord {
4389 pre: pre.clone(),
4390 post: None,
4391 lines: None,
4392 limit: None,
4393 agent: None,
4394 outcome: GraftOutcome::Running,
4395 requested: graft::requested(&plan),
4396 adoption: None,
4397 adoption_error: None,
4398 });
4399 self.state.event(
4400 "graft",
4401 format!(
4402 "porting strengths of candidate(s) {} onto {}",
4403 plan.strengths
4404 .iter()
4405 .map(|(l, _, _)| l.to_string())
4406 .collect::<Vec<_>>()
4407 .join(", "),
4408 plan.winner
4409 ),
4410 );
4411 self.state.save()?;
4412
4413 let fix_prompt = prompt::graft(
4414 &self.state.instruction,
4415 plan.winner,
4416 &plan.strengths,
4417 &self.state.config.graph.language,
4418 );
4419 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4420 let sessions = self.state.config.graph.sessions;
4421 let artifacts = agent::artifacts_dir(&self.state.dir());
4422 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4423 let (_, seat, out) = self
4424 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4425 SeatJob {
4426 prompt: fix_prompt.clone(),
4427 spec,
4428 seat,
4429 cwd: wt.clone(),
4430 timeout,
4431 allow_write: true,
4432 sessions,
4433 artifacts: artifacts.clone(),
4434 stem: "graft".to_owned(),
4435 handover: None,
4436 }
4437 })
4438 .await;
4439 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4440 self.state.graft.as_mut().expect("graft record").outcome =
4443 GraftOutcome::WaitingForRestore;
4444 return self.stop_for_resource();
4445 }
4446 let mut why: Option<String> = None;
4447 match &out {
4448 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4449 why = Some("the fixer produced nothing usable".to_owned());
4450 }
4451 AgentOutcome::Ok(_) => {}
4452 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4453 AgentOutcome::Quota(o) => {
4454 self.state.quota.push(QuotaLoss {
4455 seat: seat.key.clone(),
4456 node: "graft".to_owned(),
4457 at: Timestamp::now(),
4458 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4459 });
4460 why = Some("rate limited (quota)".to_owned());
4461 }
4462 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4463 why = Some(format!(
4464 "the fixer failed: {}",
4465 e.lines()
4466 .next()
4467 .unwrap_or("")
4468 .chars()
4469 .take(160)
4470 .collect::<String>()
4471 ));
4472 }
4473 }
4474 let agent_id = seat.agent.clone();
4475 self.state.seats.insert(seat.key.clone(), seat);
4476
4477 let mut post = None;
4478 let mut lines = None;
4479 let _coordination = self.coordinate(&wt).await?;
4480 if why.is_none() {
4481 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4482 if let Ok(r) = &rescued {
4483 self.state.note_withheld("graft", &r.withheld);
4484 }
4485 let head = git::rev_parse(&wt, "HEAD").await?;
4486 let dirty = match &rescued {
4490 Ok(r) => git::status_porcelain(&wt)
4491 .await
4492 .map(|st| {
4493 st.lines().any(|l| {
4494 !r.withheld
4495 .iter()
4496 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4497 })
4498 })
4499 .unwrap_or(true),
4500 Err(_) => true,
4501 };
4502 if dirty {
4503 why = Some("the fixer left changes that could not be committed".to_owned());
4504 } else if head == pre {
4505 why = Some("the fixer changed nothing".to_owned());
4506 } else {
4507 lines = git::changed_lines(&wt, &pre, &head).await;
4508 post = Some(head);
4509 }
4510 }
4511 let (adoption, adoption_error) = match &out {
4512 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4513 _ => (None, None),
4514 };
4515 let requested = graft::requested(&plan);
4516 let outcome = match why {
4517 None => {
4518 self.state.event(
4519 "graft",
4520 format!(
4521 "applied: {}",
4522 lines.map_or_else(
4523 || "changed lines unavailable".to_owned(),
4524 |n| format!("{n} changed lines")
4525 )
4526 ),
4527 );
4528 GraftOutcome::Applied
4529 }
4530 Some(why) => {
4531 crate::recovery::restore_repair(&wt, &pre)
4534 .await
4535 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4536 self.state.event(
4537 "graft",
4538 format!("rolled back: {why}; the winner is untouched"),
4539 );
4540 GraftOutcome::RolledBack(why)
4541 }
4542 };
4543 self.state.graft = Some(GraftRecord {
4544 pre,
4545 post,
4546 lines,
4547 limit: None,
4548 agent: Some(agent_id),
4549 outcome,
4550 requested,
4551 adoption,
4552 adoption_error,
4553 });
4554 self.state.save()?;
4555 Ok(())
4556 }
4557
4558 async fn fold_losers(&mut self) -> Result<()> {
4561 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4562 return Ok(());
4563 };
4564 if !self.state.candidate_preservation.is_empty() {
4565 bail!("candidate preservation incomplete; refusing cleanup");
4566 }
4567 let repo = self.state.repo.clone();
4568 let _coordination = self.coordinate(&repo).await?;
4569 let mut folded = Vec::new();
4570 for i in 0..self.state.candidates.len() {
4571 let c = &self.state.candidates[i];
4572 if c.label == winner || c.folded {
4573 continue;
4574 }
4575 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4576 remove_preserved_worktree(&repo, &wt).await?;
4577 if git::branch_exists(&repo, &branch).await?
4578 && !git::branch_delete(&repo, &branch).await?
4579 {
4580 bail!("candidate branch removal refused");
4581 }
4582 self.state.candidates[i].folded = true;
4583 self.state.save()?;
4584 folded.push(label.to_string());
4585 }
4586 let root = self.state.worktree_root();
4588 for j in 1..=self.roles.judges.len() {
4589 let wt = root.join(format!("judge-{j}"));
4590 if wt.exists() {
4591 remove_preserved_worktree(&repo, &wt).await?;
4592 }
4593 }
4594 if self.state.config.graph.advise {
4597 for k in 1..=self.state.config.graph.advisors {
4598 let wt = root.join(format!("advisor-{k}"));
4599 if wt.exists() {
4600 remove_preserved_worktree(&repo, &wt).await?;
4601 }
4602 }
4603 }
4604 if !folded.is_empty() {
4605 self.state
4606 .event("fold", format!("folded candidates {}", folded.join(", ")));
4607 self.state.save()?;
4608 }
4609 Ok(())
4610 }
4611
4612 async fn sync_to_base(&mut self) -> Result<()> {
4652 if self.state.status == RunStatus::AlreadyInBase {
4653 return Ok(());
4654 }
4655 let conflicted = self
4656 .state
4657 .base_sync
4658 .as_ref()
4659 .is_some_and(|s| s.conflict.is_some());
4660 let Some(winner) = self.state.winner().cloned() else {
4661 return Ok(());
4662 };
4663
4664 let repo = self.state.repo.clone();
4665 let remote = self.state.config.merge.remote.clone();
4666 let base_branch = self.state.base_branch.clone();
4667 let tracking = format!("{remote}/{base_branch}");
4668
4669 git::fetch(&repo, &remote, &base_branch).await.ok();
4670 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4675 return Ok(());
4676 };
4677
4678 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4679 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4680 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4681
4682 if (behind > 0 || conflicted || head == tip)
4691 && self
4692 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4693 .await?
4694 {
4695 return Ok(());
4696 }
4697 if conflicted {
4698 return Ok(());
4699 }
4700
4701 if behind == 0 {
4702 if let Some(from) = self
4711 .state
4712 .rebase_fixes
4713 .iter()
4714 .rev()
4715 .find_map(|r| r.from.clone())
4716 && from != head
4717 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4718 .await
4719 .is_ok_and(|o| o.ok())
4720 {
4721 git::sync_to_head(&winner.worktree).await?;
4722 }
4723 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4729 if let Some(why) = &conflict {
4730 self.state.status = RunStatus::Blocked;
4731 self.state.event("land", why.clone());
4732 }
4733 self.state.base_sync = Some(BaseSync {
4734 tip,
4735 behind: 0,
4736 attempts,
4737 conflict,
4738 already_in: None,
4739 });
4740 self.state.save()?;
4741 return Ok(());
4742 }
4743
4744 if attempts >= BASE_SYNC_ROUNDS {
4745 let why = format!(
4746 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4747 rebase(s); rebasing again would only race it",
4748 winner.branch
4749 );
4750 self.state.status = RunStatus::Blocked;
4751 self.state.base_sync = Some(BaseSync {
4752 tip,
4753 behind,
4754 attempts,
4755 conflict: Some(why.clone()),
4756 already_in: None,
4757 });
4758 self.state.event("land", why);
4759 self.state.save()?;
4760 return Ok(());
4761 }
4762
4763 self.state.event(
4764 "land",
4765 format!(
4766 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4767 winner.branch
4768 ),
4769 );
4770 self.state.save()?;
4771
4772 let branch_tracking = format!("{remote}/{}", winner.branch);
4778 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4779 let remote_tip = if self.state.task_pr_branch().is_none()
4780 && matches!(&fetched_branch, Ok(o) if o.ok())
4781 {
4782 git::rev_parse(&repo, &branch_tracking).await.ok()
4783 } else {
4784 None
4785 };
4786 if let Some(theirs) = &remote_tip
4790 && !git::is_ancestor(&repo, theirs, &head).await
4791 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4792 .await
4793 .is_ok_and(|missing| missing.is_empty())
4794 {
4795 let why = format!(
4796 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4797 them",
4798 short(theirs),
4799 winner.branch
4800 );
4801 self.state.status = RunStatus::Blocked;
4802 self.state.base_sync = Some(BaseSync {
4803 tip,
4804 behind,
4805 attempts,
4806 conflict: Some(why.clone()),
4807 already_in: None,
4808 });
4809 self.state.event("land", why);
4810 self.state.save()?;
4811 return Ok(());
4812 }
4813
4814 let scratch = self.state.dir().join("base-sync");
4815 let rebased = match crate::rebase::rebase_with_fixer(
4816 &mut self.state,
4817 &scratch,
4818 &winner.branch,
4819 &tracking,
4820 )
4821 .await
4822 {
4823 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4824 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4825 Err(e) => Err(e),
4826 };
4827 let attempts = attempts + 1;
4828 match rebased {
4829 Ok(None) => {
4830 git::sync_to_head(&winner.worktree).await?;
4834 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4835 let mut conflict = None;
4836 if let Some(pinned) = &remote_tip {
4837 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4838 match pushed {
4839 Ok(o) if o.ok() => self.state.event(
4840 "land",
4841 format!("pushed rebased {} to {remote}", winner.branch),
4842 ),
4843 Ok(o) => {
4844 conflict = Some(format!(
4845 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4846 winner.branch,
4847 short(pinned),
4848 o.stderr.chars().take(600).collect::<String>()
4849 ));
4850 }
4851 Err(e) => {
4852 conflict = Some(format!(
4853 "rebased {} locally but could not push it: {e:#}",
4854 winner.branch
4855 ));
4856 }
4857 }
4858 }
4859 if let Some(why) = &conflict {
4860 self.state.status = RunStatus::Blocked;
4861 self.state.event("land", why.clone());
4862 }
4863 self.state.base_sync = Some(BaseSync {
4864 tip: tip.clone(),
4865 behind: 0,
4866 attempts,
4867 conflict,
4868 already_in: None,
4869 });
4870 self.state
4871 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4872 }
4873 Ok(Some(conflict)) => {
4874 let why = format!(
4875 "{} conflicts with {tracking} and did not rebase: {}",
4876 winner.branch,
4877 conflict.chars().take(600).collect::<String>()
4878 );
4879 self.state.status = RunStatus::Blocked;
4880 self.state.base_sync = Some(BaseSync {
4881 tip,
4882 behind,
4883 attempts,
4884 conflict: Some(why.clone()),
4885 already_in: None,
4886 });
4887 self.state.event("land", why);
4888 }
4889 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4893 self.state.save()?;
4894 return Ok(());
4895 }
4896 Err(e) => {
4897 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4898 self.state.status = RunStatus::Blocked;
4899 self.state.base_sync = Some(BaseSync {
4900 tip,
4901 behind,
4902 attempts,
4903 conflict: Some(why.clone()),
4904 already_in: None,
4905 });
4906 self.state.event("land", why);
4907 }
4908 }
4909 self.state.save()?;
4910 Ok(())
4911 }
4912
4913 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4919 if self.state.task_pr_branch().is_some() {
4922 return None;
4923 }
4924 let repo = self.state.repo.clone();
4925 let remote = self.state.config.merge.remote.clone();
4926 let fetched = git::fetch(&repo, &remote, branch).await;
4927 if !matches!(&fetched, Ok(o) if o.ok()) {
4928 return None;
4929 }
4930 let branch_tracking = format!("{remote}/{branch}");
4931 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4932 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4933 return None;
4934 }
4935 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4936 .await
4937 .is_ok_and(|missing| missing.is_empty())
4938 {
4939 return Some(format!(
4940 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4941 them",
4942 short(&theirs)
4943 ));
4944 }
4945 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4946 Ok(o) if o.ok() => {
4947 self.state
4948 .event("land", format!("pushed rebased {branch} to {remote}"));
4949 None
4950 }
4951 Ok(o) => Some(format!(
4952 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4953 someone may have pushed): {}",
4954 short(&theirs),
4955 o.stderr.chars().take(600).collect::<String>()
4956 )),
4957 Err(e) => Some(format!(
4958 "{branch} is rebased locally but could not be pushed: {e:#}"
4959 )),
4960 }
4961 }
4962
4963 async fn settle_already_in(
4980 &mut self,
4981 branch: &str,
4982 tip: &str,
4983 head: &str,
4984 attempts: usize,
4985 behind: usize,
4986 ) -> Result<bool> {
4987 let repo = self.state.repo.clone();
4988 let remote = self.state.config.merge.remote.clone();
4989 let start = self.state.base_commit.clone();
4990 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4991 Ok(Some(e)) => e,
4992 Ok(None) => return Ok(false),
4993 Err(e) => {
4994 tracing::warn!("already-in-base check for {branch}: {e:#}");
4995 return Ok(false);
4996 }
4997 };
4998 let mut verified = vec![head.to_owned()];
4999 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5000 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5001 if matches!(&fetched, Ok(o) if o.ok())
5002 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5003 && theirs != head
5004 {
5005 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5006 Ok(Some(_)) => verified.push(theirs),
5007 _ => return Ok(false),
5008 }
5009 }
5010 let base_branch = self.state.base_branch.clone();
5011 let message = format!(
5012 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5013 land",
5014 evidence.names(),
5015 evidence.proof.as_str()
5016 );
5017 let closed =
5018 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5019 .await;
5020 match closed {
5021 Ok(Ok(url)) => self
5022 .state
5023 .event("land", format!("closed {url}: superseded on {base_branch}")),
5024 Ok(Err(why)) => self
5025 .state
5026 .event("land", format!("did not close a pull request: {why}")),
5027 Err(e) => {
5028 let why = format!(
5029 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5030 not be closed ({e:#}); resume to retry"
5031 );
5032 self.state.status = RunStatus::Blocked;
5033 self.state.base_sync = Some(BaseSync {
5034 tip: tip.to_owned(),
5035 behind,
5036 attempts,
5037 conflict: Some(why.clone()),
5038 already_in: None,
5039 });
5040 self.state.event("land", why);
5041 self.state.save()?;
5042 return Ok(true);
5043 }
5044 }
5045 self.state.status = RunStatus::AlreadyInBase;
5046 self.state.base_sync = Some(BaseSync {
5047 tip: tip.to_owned(),
5048 behind,
5049 attempts,
5050 conflict: None,
5051 already_in: Some(evidence),
5052 });
5053 self.state.event("land", message);
5054 self.state.save()?;
5055 self.settle_questions();
5056 Ok(true)
5057 }
5058
5059 fn landing_base(&self) -> String {
5069 self.state
5070 .base_sync
5071 .as_ref()
5072 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5073 }
5074
5075 pub async fn fix_selected(
5108 &mut self,
5109 ids: &[String],
5110 reason: &str,
5111 allow_stale: bool,
5112 ) -> Result<()> {
5113 let reason = reason.trim();
5114 if reason.is_empty() {
5115 bail!("a fix request needs a reason — that is the operator's own record of why");
5116 }
5117 if ids.is_empty() {
5118 bail!("no finding id given");
5119 }
5120 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5121 bail!(
5122 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5123 has already concluded — can be given a targeted fix. A run still \
5124 in progress should simply be resumed; a `merged` run's branch has \
5125 already landed, so its answer is a fresh `magi review <branch>`, \
5126 not reopening this run's own record",
5127 self.state.id,
5128 self.state.status.as_str()
5129 );
5130 }
5131 let Some(winner) = self.state.winner().cloned() else {
5132 bail!("run {} has no winning candidate to fix", self.state.id);
5133 };
5134 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5135 bail!(
5136 "branch `{}` no longer exists; this run cannot be extended",
5137 winner.branch
5138 );
5139 }
5140 let home = crate::run::home();
5141 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5142 bail!(
5143 "run {} is currently being worked on by another magi process",
5144 self.state.id
5145 );
5146 }
5147 let _claim = FixClaim::acquire(&self.state.dir())?;
5153
5154 let mut seen = BTreeSet::new();
5158 let mut findings = Vec::new();
5159 let mut missing = Vec::new();
5160 for id in ids {
5161 if !seen.insert(id.clone()) {
5162 continue;
5163 }
5164 match self.state.finding(id) {
5165 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5166 id: f.id.clone(),
5167 severity: f.severity,
5168 reviewer_vote: rec.vote,
5169 round: round.round,
5170 round_head: round.head.clone(),
5171 reviewer: rec.reviewer,
5172 agent: rec.agent.clone(),
5173 file: f.file.clone(),
5174 line: f.line,
5175 title: f.title.clone(),
5176 detail: f.detail.clone(),
5177 outcome: OperatorFixOutcome::Pending,
5178 }),
5179 None => missing.push(id.clone()),
5180 }
5181 }
5182 if !missing.is_empty() {
5183 bail!(
5184 "unknown finding id(s): {}; nothing was changed",
5185 missing.join(", ")
5186 );
5187 }
5188
5189 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5190 let stale_details: Vec<(String, String)> = findings
5191 .iter()
5192 .filter(|f| f.round_head != head_at_request)
5193 .map(|f| (f.id.clone(), f.round_head.clone()))
5194 .collect();
5195 let stale = !stale_details.is_empty();
5196 if stale && !allow_stale {
5197 bail!(
5198 "the branch has moved since some finding(s) were raised — {} — now \
5199 at {}; pass --allow-stale to fix anyway, or re-run review first",
5200 stale_details
5201 .iter()
5202 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5203 .collect::<Vec<_>>()
5204 .join(", "),
5205 short(&head_at_request)
5206 );
5207 }
5208
5209 let request = OperatorFixRequest {
5210 requested_at: Timestamp::now(),
5211 reason: reason.to_owned(),
5212 findings,
5213 head_at_request: head_at_request.clone(),
5214 allow_stale,
5215 stale,
5216 fix: None,
5217 result_head: None,
5218 follow_up_review_run: None,
5219 };
5220 self.state.event(
5221 "fix",
5222 format!(
5223 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5224 request.findings.len(),
5225 request
5226 .findings
5227 .iter()
5228 .map(|f| f.id.as_str())
5229 .collect::<Vec<_>>()
5230 .join(", "),
5231 ),
5232 );
5233 self.state.operator_fixes.push(request);
5240 self.state.save()?;
5241 let request_index = self.state.operator_fixes.len() - 1;
5242
5243 let repo = self.state.repo.clone();
5252 let coordination = self.coordinate(&repo).await?;
5253 if winner.worktree.exists() {
5254 let dirty = git::git(
5257 &winner.worktree,
5258 &["status", "--porcelain", "--untracked-files=all"],
5259 )
5260 .await?;
5261 let only_withheld = dirty.lines().all(|l| {
5262 l.strip_prefix("?? ")
5263 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5264 });
5265 if !only_withheld {
5266 bail!(
5267 "`{}` has uncommitted changes; refusing to touch it — commit or \
5268 discard them first",
5269 winner.worktree.display()
5270 );
5271 }
5272 git::worktree_remove(&self.state.repo, &winner.worktree)
5273 .await
5274 .ok();
5275 }
5276 let fix_worktree = self.state.worktree_root().join("operator-fix");
5277 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5278 git::git(
5279 &self.state.repo,
5280 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5281 )
5282 .await
5283 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5284 if !git::is_clean(&fix_worktree).await? {
5285 git::worktree_remove(&self.state.repo, &fix_worktree)
5286 .await
5287 .ok();
5288 bail!(
5289 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5290 winner.branch
5291 );
5292 }
5293 let repo_for_setup = self.state.repo.clone();
5294 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5295 Ok(()) => {
5296 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5297 }
5298 Err(e) => Err(e),
5299 };
5300 if let Err(e) = setup {
5301 git::worktree_remove(&self.state.repo, &fix_worktree)
5302 .await
5303 .ok();
5304 return Err(e);
5305 }
5306
5307 let run_id = self.state.id.clone();
5308 let prompts = self.state.config.prompts.clone();
5309 let language = self.state.config.graph.language.clone();
5310 let sessions = self.state.config.graph.sessions;
5311 let artifacts = agent::artifacts_dir(&self.state.dir());
5312 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5313 .findings
5314 .iter()
5315 .map(|f| Finding {
5316 id: f.id.clone(),
5317 severity: f.severity,
5318 file: f.file.clone(),
5319 line: f.line,
5320 title: f.title.clone(),
5321 detail: f.detail.clone(),
5322 })
5323 .collect();
5324 let fix_prompt = prompt::operator_fix_in(
5325 &self.state.instruction,
5326 &finding_list,
5327 reason,
5328 &stale_details,
5329 &head_at_request,
5330 &language,
5331 &self.state.config.graph.github_language,
5332 );
5333 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5334 drop(coordination);
5335 let (job, seat, out) = self
5336 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5337 prompt: fix_prompt.clone(),
5338 spec,
5339 seat,
5340 cwd: fix_worktree.clone(),
5341 timeout,
5342 allow_write: true,
5343 sessions,
5344 artifacts: artifacts.clone(),
5345 stem: "operator-fix".to_owned(),
5346 handover: None,
5347 })
5348 .await;
5349 let agent_id = seat.agent.clone();
5350
5351 let mut fix = FixRecord {
5352 agent: agent_id,
5353 addressed: Vec::new(),
5354 rejected: Vec::new(),
5355 notes: String::new(),
5356 committed: false,
5357 failed: None,
5358 duration_ms: 0,
5359 continuation: None,
5360 };
5361 let mut final_seat = seat.clone();
5362 match out {
5363 AgentOutcome::Ok(o) => {
5364 fix.duration_ms = o.duration_ms;
5365 let parsed = verdict::extract_json::<FixReport>(&o.text);
5366 let incomplete_reason = match &parsed {
5367 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5368 "the reply parsed, but it reported a command whose own CLI \
5369 never confirmed an exit status"
5370 .to_owned(),
5371 ),
5372 Ok(_) => None,
5373 Err(e) => Some(e.to_string()),
5374 };
5375 match incomplete_reason {
5376 None => {
5377 let report = parsed.expect("checked Ok above");
5378 fix.addressed = report.addressed;
5379 fix.rejected = report.rejected;
5380 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5381 }
5382 Some(reason) => {
5383 let (resumed_seat, resolved, failure, cont) = self
5384 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5385 .await;
5386 fix.duration_ms += cont.cumulative_wait_ms;
5387 fix.continuation = Some(cont);
5388 final_seat = resumed_seat;
5389 match resolved {
5390 Some(report) => {
5391 fix.addressed = report.addressed;
5392 fix.rejected = report.rejected;
5393 fix.notes =
5394 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5395 }
5396 None => fix.failed = failure,
5397 }
5398 }
5399 }
5400 }
5401 AgentOutcome::Dropped(o) => {
5402 fix.duration_ms = o.duration_ms;
5403 let why = o
5404 .dropped
5405 .as_ref()
5406 .map(|d| d.why.as_str())
5407 .unwrap_or("the CLI ended the stream without delivering its answer");
5408 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5409 }
5410 AgentOutcome::Quota(o) => {
5411 self.state.quota.push(QuotaLoss {
5412 seat: final_seat.key.clone(),
5413 node: "fix".to_owned(),
5414 at: Timestamp::now(),
5415 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5416 });
5417 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5418 }
5419 AgentOutcome::Failed(e) => fix.failed = Some(e),
5420 AgentOutcome::ResourceBlocked(e) => {
5421 fix.failed = Some(format!("the fixer was not started: {e}"));
5422 }
5423 }
5424 if fix.continuation.is_none() {
5425 fix.continuation = Some(ContinuationRecord::not_needed());
5426 }
5427 self.state.seats.insert(final_seat.key.clone(), final_seat);
5428
5429 let rescue_message = format!(
5430 "magi: operator-selected fix ({}) (uncommitted work)",
5431 self.state.operator_fixes[request_index]
5432 .findings
5433 .iter()
5434 .map(|f| f.id.as_str())
5435 .collect::<Vec<_>>()
5436 .join(", ")
5437 );
5438 let repo = self.state.repo.clone();
5439 let coordination = self.coordinate(&repo).await?;
5440 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5441 self.state.note_withheld("fix", &r.withheld);
5442 }
5443 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5444 fix.committed = after != head_at_request;
5445 git::worktree_remove(&self.state.repo, &fix_worktree)
5446 .await
5447 .ok();
5448 drop(coordination);
5449
5450 self.state.event(
5451 "fix",
5452 match &fix.failed {
5453 Some(reason) => format!(
5454 "operator fix: adoption report was lost ({reason}); {}",
5455 if fix.committed {
5456 "committed"
5457 } else {
5458 "NO new commit"
5459 }
5460 ),
5461 None => format!(
5462 "operator fix: {} addressed, {} rejected, {}",
5463 fix.addressed.len(),
5464 fix.rejected.len(),
5465 if fix.committed {
5466 "committed"
5467 } else {
5468 "NO new commit"
5469 }
5470 ),
5471 },
5472 );
5473
5474 for f in &mut self.state.operator_fixes[request_index].findings {
5481 f.outcome = if fix.failed.is_some() {
5482 OperatorFixOutcome::Unreported
5483 } else if fix.addressed.contains(&f.id) {
5484 OperatorFixOutcome::Addressed
5485 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5486 OperatorFixOutcome::Rejected { why: r.why.clone() }
5487 } else {
5488 OperatorFixOutcome::Unreported
5489 };
5490 }
5491
5492 let committed = fix.committed;
5493 if committed {
5494 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5495 }
5496 self.state.operator_fixes[request_index].fix = Some(fix);
5497 self.state.save()?;
5500
5501 if committed {
5502 self.state.event(
5503 "fix",
5504 format!(
5505 "operator fix committed {}; opening a follow-up review-only run",
5506 short(&after)
5507 ),
5508 );
5509 let origin =
5512 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5513 match Self::review(
5514 &self.state.repo,
5515 &winner.branch,
5516 self.state.config.clone(),
5517 origin,
5518 )
5519 .await
5520 {
5521 Ok(mut follow_up) => {
5522 follow_up.state.event(
5523 "start",
5524 format!(
5525 "requested by an operator fix on run {} for finding(s) {}",
5526 self.state.id,
5527 self.state.operator_fixes[request_index]
5528 .findings
5529 .iter()
5530 .map(|f| f.id.as_str())
5531 .collect::<Vec<_>>()
5532 .join(", "),
5533 ),
5534 );
5535 follow_up.state.save()?;
5536 let follow_up_id = follow_up.state.id.clone();
5537 let adopted = match crate::direct::adopt(
5540 &follow_up.state,
5541 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5542 ) {
5543 Ok(a) => a,
5544 Err(e) => {
5545 self.state.event(
5548 "fix",
5549 format!(
5550 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5551 ),
5552 );
5553 self.state.save()?;
5554 return Ok(());
5555 }
5556 };
5557 let executed = follow_up.execute().await;
5558 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5559 if let Some(a) = adopted {
5560 a.finish(&follow_up.state, executed);
5561 }
5562 if let Some(e) = failure {
5563 self.state.event(
5564 "fix",
5565 format!(
5566 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5567 ),
5568 );
5569 }
5570 self.state.operator_fixes[request_index].follow_up_review_run =
5571 Some(follow_up_id);
5572 }
5573 Err(e) => {
5574 self.state.event(
5575 "fix",
5576 format!("committed the fix but could not open a follow-up review: {e:#}"),
5577 );
5578 }
5579 }
5580 self.state.save()?;
5581 }
5582
5583 Ok(())
5584 }
5585
5586 async fn ask_fixer(
5606 &mut self,
5607 winner: &Candidate,
5608 node: &'static str,
5609 round: Option<usize>,
5610 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5611 ) -> (SeatJob, SeatState, AgentOutcome) {
5612 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5613 self.ask_fixer_among(attempts, node, round, None, build)
5614 .await
5615 }
5616
5617 async fn ask_fixer_among(
5622 &mut self,
5623 attempts: Vec<(AgentSpec, String)>,
5624 node: &'static str,
5625 round: Option<usize>,
5626 restore: Option<(&Path, &str)>,
5627 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5628 ) -> (SeatJob, SeatState, AgentOutcome) {
5629 let run_id = self.state.id.clone();
5630 let prompts = self.state.config.prompts.clone();
5631 let cache = self.state.config.cache_dir();
5632 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5633 let mut last = None;
5634 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5635 let seat = self.seat(&key, &spec.id);
5636 let mut job = build(spec.clone(), seat);
5637 if i > 0 {
5638 job.stem = format!("{}-{}", job.stem, spec.id);
5639 }
5640 let ctx = WaveCtx {
5641 carry_seats: false,
5642 pauses: Some((&self.pause, &self.interrupt)),
5643 run: &run_id,
5644 node,
5645 prompts: &prompts,
5646 cache: cache.as_deref(),
5647 round,
5648 };
5649 let (seat, out) =
5650 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5651 let advances = match &out {
5654 AgentOutcome::Ok(o) => agent::output_advances(o),
5655 AgentOutcome::ResourceBlocked(_) => false,
5656 _ => true,
5657 };
5658 if let Some(next) = ids.get(i + 1)
5659 && advances
5660 {
5661 self.state.seats.insert(seat.key.clone(), seat.clone());
5662 let class =
5663 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5664 let reason = match &out {
5665 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5666 other => fail_reason(other),
5667 };
5668 if let Some((wt, rev)) = restore
5672 && let Err(e) = self.graft_restore(wt, rev).await
5673 {
5674 let why =
5675 format!("could not restore the worktree before the next fixer: {e:#}");
5676 tracing::warn!("{why}");
5677 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5678 AgentOutcome::ResourceBlocked(why)
5679 } else {
5680 AgentOutcome::Failed(why)
5681 };
5682 last = Some((job, seat, out));
5683 break;
5684 }
5685 record_handover(
5686 &mut self.state,
5687 node,
5688 &seat.key,
5689 &spec.id,
5690 next,
5691 &class,
5692 &reason,
5693 );
5694 continue;
5695 }
5696 last = Some((job, seat, out));
5697 break;
5698 }
5699 last.expect("the fixer chain always has an entry")
5700 }
5701
5702 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5713 use crate::recovery::{Mode, Repair};
5714 if !self.state.reviews.is_empty() {
5715 return Ok(());
5716 }
5717 let Some(mut rec) = self.state.recovery.clone() else {
5718 return Ok(());
5719 };
5720 if rec.mode != Mode::VerifyRepair {
5721 return Ok(());
5722 }
5723 let wt = winner.worktree.clone();
5724 let mut head = git::rev_parse(&wt, "HEAD").await?;
5725 match rec.repair.clone() {
5726 Repair::Done { .. } => return Ok(()),
5727 Repair::WaitingForRestore { before } => {
5728 self.graft_restore(&wt, &before).await?;
5729 head = before;
5730 rec.repair = Repair::NotStarted;
5731 self.state.recovery = Some(rec.clone());
5732 self.state.save()?;
5733 }
5734 Repair::Started { before } => {
5735 if let Err(e) = self.graft_restore(&wt, &before).await {
5739 if e.is::<crate::repo_lock::ResourceBlocked>() {
5740 return Err(e);
5741 }
5742 return self
5743 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5744 .await;
5745 }
5746 rec.repair = Repair::Done {
5747 before: before.clone(),
5748 after: before,
5749 committed: false,
5750 failed: Some("interrupted; rolled back".to_owned()),
5751 };
5752 self.state
5753 .event("recovery", "the repair was interrupted and rolled back");
5754 self.state.recovery = Some(rec);
5755 self.state.save()?;
5756 return Ok(());
5757 }
5758 Repair::NotStarted => {}
5759 }
5760 if crate::recovery::recheck(&mut rec, &head) {
5761 self.state.event("recovery", rec.summary());
5762 self.state.recovery = Some(rec);
5763 self.state.save()?;
5764 return Ok(());
5765 }
5766 rec.repair = Repair::Started {
5767 before: head.clone(),
5768 };
5769 self.state.recovery = Some(rec.clone());
5770 self.state.save()?;
5771
5772 let language = self.state.config.graph.language.clone();
5773 let sessions = self.state.config.graph.sessions;
5774 let artifacts = agent::artifacts_dir(&self.state.dir());
5775 let fix_prompt = prompt::recovery_fix(
5776 &self.state.instruction,
5777 &rec,
5778 self.state.config.graph.review_rounds,
5779 &language,
5780 );
5781 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5782 self.state.event(
5783 "recovery",
5784 format!(
5785 "repair: the fixer starts from the failed verification on {}",
5786 short(&head)
5787 ),
5788 );
5789 let (_job, seat, out) = self
5790 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5791 prompt: fix_prompt.clone(),
5792 spec,
5793 seat,
5794 cwd: wt.clone(),
5795 timeout,
5796 allow_write: true,
5797 sessions,
5798 artifacts: artifacts.clone(),
5799 stem: "fix-recovery".to_owned(),
5800 handover: None,
5801 })
5802 .await;
5803 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5804 rec.repair = Repair::WaitingForRestore { before: head };
5805 self.state.recovery = Some(rec);
5806 return self.stop_for_resource();
5807 }
5808 let failed = match &out {
5809 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5810 AgentOutcome::Quota(o) => {
5811 self.state.quota.push(QuotaLoss {
5812 seat: seat.key.clone(),
5813 node: "fix".to_owned(),
5814 at: Timestamp::now(),
5815 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5816 });
5817 Some("rate limited (quota); the fixer could not run".to_owned())
5818 }
5819 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5820 AgentOutcome::Failed(e) => Some(e.clone()),
5821 };
5822 self.state.seats.insert(seat.key.clone(), seat);
5823 let coordination = self.coordinate(&wt).await?;
5824 let failed = match failed {
5825 Some(why) => {
5827 if let Err(e) = git::restore_to(&wt, &head).await {
5828 return self
5829 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5830 .await;
5831 }
5832 Some(why)
5833 }
5834 None => {
5835 if let Ok(r) = git::rescue_commit(
5836 &wt,
5837 "magi: recovery repair of the failed verification (uncommitted work)",
5838 )
5839 .await
5840 {
5841 self.state.note_withheld("fix", &r.withheld);
5842 }
5843 None
5844 }
5845 };
5846 let after = git::rev_parse(&wt, "HEAD").await?;
5847 drop(coordination);
5848 self.state.event(
5849 "recovery",
5850 match &failed {
5851 Some(why) => format!("repair: nothing kept ({why})"),
5852 None if after != head => {
5853 format!("repair: committed {}", short(&after))
5854 }
5855 None => "repair: the fixer made no commit".to_owned(),
5856 },
5857 );
5858 rec.repair = Repair::Done {
5859 before: head.clone(),
5860 committed: after != head,
5861 after,
5862 failed,
5863 };
5864 self.state.recovery = Some(rec);
5865 self.state.save()?;
5866 Ok(())
5867 }
5868
5869 async fn stop_recovery(
5872 &mut self,
5873 mut rec: crate::recovery::RecoveryRecord,
5874 head: &str,
5875 why: String,
5876 ) -> Result<()> {
5877 rec.repair = crate::recovery::Repair::Started {
5879 before: head.to_owned(),
5880 };
5881 self.state.recovery = Some(rec);
5882 self.state
5883 .event("recovery", format!("repair could not be undone: {why}"));
5884 self.state.status = RunStatus::Blocked;
5885 self.state.save()?;
5886 bail!("recovery repair could not be rolled back: {why}")
5887 }
5888
5889 async fn review_loop(&mut self) -> Result<()> {
5890 loop {
5891 if self
5892 .state
5893 .base_sync
5894 .as_ref()
5895 .is_some_and(|s| s.conflict.is_some())
5896 {
5897 return Ok(());
5898 }
5899 if self.catchup_repairable() {
5900 let record = self.state.reviews.last().expect("repairable round").clone();
5901 let winner = self
5902 .state
5903 .winner()
5904 .cloned()
5905 .context("catch-up repair has no winner")?;
5906 if !self.fix_review_round(&winner, record).await? {
5907 return Ok(());
5908 }
5909 }
5910 self.review_loop_pass().await?;
5911 if !self.catchup_repairable() {
5912 return Ok(());
5913 }
5914 }
5915 }
5916
5917 fn catchup_repairable(&self) -> bool {
5920 self.state.reviews.len() < self.state.config.graph.review_rounds
5921 && self.state.reviews.last().is_some_and(|r| {
5922 !r.clean
5923 && r.blocking == 0
5924 && r.fix.is_none()
5925 && r.e2e_status() == E2eStatus::Failed
5926 })
5927 }
5928
5929 async fn review_loop_pass(&mut self) -> Result<()> {
5930 if self
5935 .state
5936 .base_sync
5937 .as_ref()
5938 .is_some_and(|s| s.conflict.is_some())
5939 {
5940 return Ok(());
5941 }
5942 let run_id = self.state.id.clone();
5947 let prompts = self.state.config.prompts.clone();
5948 let Some(winner) = self.state.winner().cloned() else {
5949 return Ok(());
5950 };
5951 let max_rounds = self.state.config.graph.review_rounds;
5952 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5962 if status == RunStatus::Gating {
5965 self.record_contested_handoff();
5966 }
5967 self.state.status = status;
5968 self.state.save()?;
5969 return Ok(());
5970 }
5971 self.state.status = RunStatus::Reviewing;
5972 if self.state.reviews.last().is_some_and(|r| {
5982 r.e2e_status() == E2eStatus::ResourceBlocked
5983 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5984 }) {
5985 let shell = self.state.config.shell();
5986 return self
5987 .stop_unresolved_review(
5988 "the last round's own verification never resolved",
5989 &shell,
5990 &winner.worktree,
5991 )
5992 .await;
5993 }
5994
5995 self.recovery_repair(&winner).await?;
5996 if self.resource_halt()? {
5997 return Ok(());
5998 }
5999
6000 let repo = self.state.repo.clone();
6001 let root = self.state.worktree_root();
6002 let language = self.state.config.graph.language.clone();
6003 let sessions = self.state.config.graph.sessions;
6004 let artifacts = self.merge_repair_artifacts();
6005 let base = self.landing_base();
6006 let base_short = short(&base);
6007 let reviewers = self.roles.reviewers.clone();
6008 let shell = self.state.config.shell();
6009
6010 for round in (self.state.reviews.len() + 1)..=max_rounds {
6011 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6012 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6013 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6014 let prev_verification = self
6023 .state
6024 .reviews
6025 .last()
6026 .and_then(|r| r.verification_summary(&head));
6027 let prior_section = match (&self.state.recovery, round) {
6030 (Some(rec), 1) if self.state.reviews.is_empty() => {
6031 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6032 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6033 Some(crate::recovery::head_of(
6034 &git::diff(&winner.worktree, reviewed, "HEAD")
6035 .await
6036 .unwrap_or_default(),
6037 crate::recovery::SINCE_DIFF_MAX,
6038 ))
6039 }
6040 _ => None,
6041 };
6042 Some(prompt::recovery_review(rec, since.as_deref()))
6043 }
6044 _ => None,
6045 };
6046
6047 let mut jobs = Vec::new();
6051 let coordination = self.coordinate(&repo).await?;
6052 for (r, spec) in reviewers.iter().cloned().enumerate() {
6053 let wt = root.join(format!("review-{}", r + 1));
6054 if wt.exists() {
6057 git::reset_detached(&wt, &head).await?;
6058 } else {
6059 git::worktree_add_detached(&repo, &wt, &head).await?;
6060 }
6061 self.setup_seat_worktree(&repo, &wt, false).await?;
6062 let seat_key = format!("review-{}", r + 1);
6063 let spec = pick_start_spec(
6066 &self.roles.reviewer_roster,
6067 spec,
6068 self.state.seat_history.get(&seat_key),
6069 );
6070 let seat = self.seat(&seat_key, &spec.id);
6071 jobs.push(SeatJob {
6072 prompt: prompt::review(&prompt::ReviewCtx {
6073 instruction: &self.state.instruction,
6074 branch: &winner.branch,
6075 base_short: &base_short,
6076 stat: &stat,
6077 patch: &patch,
6078 verification: prev_verification.as_ref(),
6079 prior: prior_section.as_deref(),
6080 reviewers: reviewers.len(),
6081 round,
6082 rounds: max_rounds,
6083 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6086 lens: Lens::for_seat(r),
6087 language: &language,
6088 github_language: &self.state.config.graph.github_language,
6089 }),
6090 spec,
6091 seat,
6092 cwd: wt,
6093 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6094 allow_write: false,
6095 sessions,
6096 artifacts: artifacts.clone(),
6097 stem: format!("review-{round}-{}", r + 1),
6098 handover: None,
6099 });
6100 }
6101
6102 drop(coordination);
6103 self.state.event(
6104 "review",
6105 format!(
6106 "round {round}: {} reviewers on {}",
6107 jobs.len(),
6108 short(&head)
6109 ),
6110 );
6111 let mut quota_losses = Vec::new();
6112 let review_retries = self.state.config.graph.retries;
6113 let review_cache = self.state.config.cache_dir();
6114 let ctx = WaveCtx {
6115 carry_seats: true,
6116 pauses: Some((&self.pause, &self.interrupt)),
6117 run: &run_id,
6118 node: "review",
6119 prompts: &prompts,
6120 cache: review_cache.as_deref(),
6121 round: Some(round),
6122 };
6123 let results = ask_json_wave::<Review>(
6124 jobs,
6125 Arc::clone(&self.sem),
6126 review_retries,
6127 &self.roles.reviewer_roster,
6128 &ctx,
6129 &mut quota_losses,
6130 &mut self.state,
6131 &|_: &Review| Ok(()),
6132 )
6133 .await;
6134 if self.state.resource_block.is_some() {
6135 return self.stop_for_resource();
6136 }
6137 let round_quota_missing = quota_losses.len();
6141 self.state.quota.extend(quota_losses);
6142
6143 let mut records = Vec::new();
6144 let mut all_findings = Vec::new();
6145 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6146 let agent_id = seat.agent.clone();
6147 self.state.seats.insert(seat.key.clone(), seat);
6148 let mut record = ReviewRecord {
6149 reviewer: r + 1,
6150 agent: agent_id,
6151 summary: String::new(),
6152 findings: Vec::new(),
6153 vote: None,
6154 failed: None,
6155 duration_ms: 0,
6156 attempts,
6162 };
6163 match res {
6164 Ok((review, out)) => {
6165 record.summary =
6173 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6174 record.vote = Some(review.vote);
6175 record.duration_ms = out.duration_ms;
6176 for (n, mut f) in review.findings.into_iter().enumerate() {
6177 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6180 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6181 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6182 f.file = f
6188 .file
6189 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6190 all_findings.push(f.clone());
6191 record.findings.push(f);
6192 }
6193 self.state.event(
6194 "review",
6195 format!(
6196 "round {round}: reviewer {} voted {} with {} finding(s)",
6197 r + 1,
6198 review.vote.label(),
6199 record.findings.len()
6200 ),
6201 );
6202 }
6203 Err(e) => {
6204 record.failed = Some(e.to_string());
6205 self.state.event(
6206 "review",
6207 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6208 );
6209 }
6210 }
6211 records.push(record);
6212 }
6213
6214 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6221 let vote_split =
6222 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6223 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6224 if vote_split {
6225 self.state.event(
6226 "review",
6227 format!(
6228 "round {round}: votes split ({}) — one round of reconsideration",
6229 initial_votes
6230 .iter()
6231 .map(|v| v.label())
6232 .collect::<Vec<_>>()
6233 .join(", ")
6234 ),
6235 );
6236 let panel: Vec<ReviewSeatReport<'_>> = records
6239 .iter()
6240 .filter_map(|r| {
6241 r.vote.map(|vote| ReviewSeatReport {
6242 reviewer: r.reviewer,
6243 vote,
6244 summary: &r.summary,
6245 findings: &r.findings,
6246 })
6247 })
6248 .collect();
6249
6250 let mut jobs = Vec::new();
6251 let mut seats_at = Vec::new();
6252 for (r, spec) in reviewers.iter().cloned().enumerate() {
6253 if records[r].vote.is_none() {
6257 continue;
6258 }
6259 let wt = root.join(format!("review-{}", r + 1));
6260 let seat_key = format!("review-{}", r + 1);
6261 let spec = self.occupant(&seat_key, spec);
6262 let seat = self.seat(&seat_key, &spec.id);
6263 let build = |with_patch: bool| {
6271 prompt::review_reconsider(&ReviewReconsiderCtx {
6272 instruction: &self.state.instruction,
6273 reviewer: r + 1,
6274 lens: Lens::for_seat(r),
6275 panel: &panel,
6276 patch: with_patch.then_some(ReviewPatch {
6277 branch: &winner.branch,
6278 base_short: &base_short,
6279 stat: &stat,
6280 patch: &patch,
6281 }),
6282 round,
6283 rounds: max_rounds,
6284 language: &language,
6285 })
6286 };
6287 let full = build(true);
6288 let prompt = if has_context(&spec, &seat, sessions) {
6289 build(false)
6290 } else {
6291 full.clone()
6292 };
6293 jobs.push(SeatJob {
6294 prompt,
6295 spec,
6296 seat,
6297 cwd: wt,
6298 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6299 allow_write: false,
6300 sessions,
6301 artifacts: artifacts.clone(),
6302 stem: format!("review-{round}-reconsider-{}", r + 1),
6303 handover: Some(full),
6304 });
6305 seats_at.push(r);
6306 }
6307
6308 let mut recon_quota_losses = Vec::new();
6309 let recon_cache = self.state.config.cache_dir();
6310 let recon_ctx = WaveCtx {
6311 carry_seats: true,
6312 pauses: Some((&self.pause, &self.interrupt)),
6313 run: &run_id,
6314 node: "review",
6315 prompts: &prompts,
6316 cache: recon_cache.as_deref(),
6317 round: Some(round),
6318 };
6319 let recon_results = ask_json_wave::<ReviewRevote>(
6320 jobs,
6321 Arc::clone(&self.sem),
6322 review_retries,
6323 &self.roles.reviewer_roster,
6324 &recon_ctx,
6325 &mut recon_quota_losses,
6326 &mut self.state,
6327 &|_: &ReviewRevote| Ok(()),
6328 )
6329 .await;
6330 if self.state.resource_block.is_some() {
6331 return self.stop_for_resource();
6332 }
6333 self.state.quota.extend(recon_quota_losses);
6334
6335 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6336 let agent_id = seat.agent.clone();
6337 self.state.seats.insert(seat.key.clone(), seat);
6338 let mut rec = ReviewRevoteRecord {
6339 reviewer: r + 1,
6340 agent: agent_id,
6341 vote: None,
6342 reason: String::new(),
6343 failed: None,
6344 };
6345 match res {
6346 Ok((rv, _)) => {
6347 rec.vote = Some(rv.vote);
6348 rec.reason =
6349 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6350 self.state.event(
6351 "review",
6352 format!(
6353 "round {round}: reviewer {} revoted {}",
6354 r + 1,
6355 rv.vote.label()
6356 ),
6357 );
6358 }
6359 Err(e) => {
6360 rec.failed = Some(e.to_string());
6361 self.state.event(
6362 "review",
6363 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6364 );
6365 }
6366 }
6367 reconsideration.push(rec);
6368 }
6369 } else if initial_votes.len() > 1 {
6370 self.state.event(
6371 "review",
6372 format!(
6373 "round {round}: votes agreed ({}) — no reconsideration",
6374 initial_votes[0].label()
6375 ),
6376 );
6377 }
6378
6379 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6380 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6381 let defer_e2e =
6392 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6393 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6394 let reason =
6395 format!("{blocking} blocking finding(s) already required a fix this round");
6396 self.state.event(
6397 "verify",
6398 format!(
6399 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6400 {}); it will run once a round has none left",
6401 short(&head)
6402 ),
6403 );
6404 (Vec::new(), false, true, Some(reason))
6405 } else {
6406 let e2e_commands = self.state.config.verify.e2e.clone();
6407 let cache_dir = self.state.config.cache_dir();
6408 let context = format!("round {round}");
6409 let wait = self.repository_wait();
6410 let (e2e, verify_retried) = with_cache_lease(
6411 &mut self.state,
6412 cache_dir.as_deref(),
6413 "e2e",
6414 "e2e",
6415 &winner.worktree,
6416 &head,
6417 verify_timeout,
6418 &context,
6419 &wait,
6420 |state, budget| {
6421 let shell = shell.clone();
6422 let e2e_commands = e2e_commands.clone();
6423 let worktree = winner.worktree.clone();
6424 let context = context.clone();
6425 async move {
6426 run_e2e_with_retry(
6427 state,
6428 &shell,
6429 &e2e_commands,
6430 &worktree,
6431 budget,
6432 &context,
6433 )
6434 .await
6435 }
6436 },
6437 )
6438 .await?;
6439 (e2e, verify_retried, false, None)
6440 };
6441
6442 let expected = records.len();
6443 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6444 let incomplete = answered < expected;
6445 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6446 let policy = self.state.config.graph.incomplete_review;
6447 let clean = round_is_clean(
6448 blocking,
6449 e2e_ok,
6450 answered,
6451 expected,
6452 round_quota_missing,
6453 policy,
6454 );
6455
6456 let mut round_record = ReviewRound {
6457 round,
6458 head: head.clone(),
6459 verified_head: None,
6460 verified_at: None,
6461 reviews: records,
6462 e2e,
6463 verify_retried,
6464 e2e_deferred,
6465 e2e_defer_reason,
6466 fix: None,
6467 blocking,
6468 answered,
6469 expected,
6470 clean,
6471 progressed: false,
6472 vote_split,
6473 reconsideration,
6474 verdict: None,
6475 };
6476 round_record.verdict = ReviewVote::worst(
6480 round_record
6481 .final_votes()
6482 .into_iter()
6483 .map(|(_, _, vote)| vote),
6484 );
6485 if !matches!(
6496 round_record.e2e_status(),
6497 E2eStatus::Deferred | E2eStatus::NotConfigured
6498 ) {
6499 round_record.verified_head = Some(head.clone());
6500 round_record.verified_at = Some(Timestamp::now());
6501 }
6502
6503 if incomplete {
6504 let missing: Vec<String> = round_record
6505 .reviews
6506 .iter()
6507 .filter(|r| r.failed.is_some())
6508 .map(|r| format!("review-{}", r.reviewer))
6509 .collect();
6510 self.state.event(
6511 "review",
6512 format!(
6513 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6514 missing.join(", ")
6515 ),
6516 );
6517 }
6518
6519 if clean {
6520 self.state.event(
6521 "review",
6522 if incomplete && policy == IncompleteReviewPolicy::Warn {
6523 format!(
6524 "round {round}: clean (warn policy, incomplete panel) — no \
6525 blocking findings from the seats that answered, verification green"
6526 )
6527 } else if incomplete {
6528 format!(
6529 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6530 quorum) — no blocking findings from the seats that answered, \
6531 verification green",
6532 expected - answered
6533 )
6534 } else {
6535 format!("round {round}: clean — no blocking findings, verification green")
6536 },
6537 );
6538 self.state.reviews.push(round_record);
6539 self.state.status = RunStatus::Gating;
6540 self.state.save()?;
6541 return Ok(());
6542 }
6543
6544 if incomplete && blocking == 0 && e2e_ok {
6552 self.state.reviews.push(round_record);
6553 self.state.save()?;
6554 if round == max_rounds {
6555 self.state.status = RunStatus::Blocked;
6556 self.state.event(
6557 "review",
6558 format!(
6559 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6560 refusing to call it clean",
6561 expected - answered
6562 ),
6563 );
6564 return Ok(());
6565 }
6566 continue;
6567 }
6568
6569 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6581 self.state.reviews.push(round_record);
6582 return self
6583 .stop_unresolved_review(
6584 "the round's own verification could not run",
6585 &shell,
6586 &winner.worktree,
6587 )
6588 .await;
6589 }
6590
6591 if round == max_rounds {
6592 self.state.reviews.push(round_record);
6593 return self
6594 .stop_reviewing(
6595 &format!(
6596 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6597 ),
6598 &shell,
6599 &winner.worktree,
6600 )
6601 .await;
6602 }
6603
6604 if !self.fix_review_round(&winner, round_record).await? {
6605 return Ok(());
6606 }
6607 }
6608 Ok(())
6609 }
6610
6611 async fn fix_review_round(
6614 &mut self,
6615 winner: &Candidate,
6616 mut round_record: ReviewRound,
6617 ) -> Result<bool> {
6618 let round = round_record.round;
6619 let shell = self.state.config.shell();
6620 let step = self.run_fix_round(winner, &mut round_record).await?;
6621 if matches!(step, FixStep::Resource) {
6622 self.stop_for_resource()?;
6623 return Ok(false);
6624 }
6625 if self.state.reviews.len() < round {
6626 self.state.reviews.push(round_record);
6627 } else {
6628 self.state.reviews[round - 1] = round_record;
6629 }
6630 self.state.save()?;
6631
6632 if matches!(step, FixStep::Recorded { report_lost: true }) {
6645 self.stop_reviewing(
6646 "the fixer's adoption report never came back, even after resuming its \
6647 own seat; refusing to start another round against the same worktree \
6648 while that is unresolved",
6649 &shell,
6650 &winner.worktree,
6651 )
6652 .await?;
6653 return Ok(false);
6654 }
6655
6656 let streak = self
6657 .state
6658 .reviews
6659 .iter()
6660 .rev()
6661 .take_while(|r| !r.progressed)
6662 .count();
6663 if streak >= STAGNANT_LIMIT {
6664 self.stop_reviewing(
6665 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6666 &shell,
6667 &winner.worktree,
6668 )
6669 .await?;
6670 return Ok(false);
6671 }
6672 Ok(true)
6673 }
6674
6675 async fn run_fix_round(
6677 &mut self,
6678 winner: &Candidate,
6679 round_record: &mut ReviewRound,
6680 ) -> Result<FixStep> {
6681 let round = round_record.round;
6682 let max_rounds = self.state.config.graph.review_rounds;
6683 let run_id = self.state.id.clone();
6684 let prompts = self.state.config.prompts.clone();
6685 let language = self.state.config.graph.language.clone();
6686 let sessions = self.state.config.graph.sessions;
6687 let artifacts = self.merge_repair_artifacts();
6688 let base = self.landing_base();
6689 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6690 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6691 let this_round_verification = round_record.verification_summary(&head);
6692 let all_findings: Vec<_> = round_record
6693 .reviews
6694 .iter()
6695 .flat_map(|r| r.findings.iter().cloned())
6696 .collect();
6697 let blocking_findings: Vec<_> = all_findings
6700 .iter()
6701 .filter(|f| f.severity.blocks())
6702 .cloned()
6703 .collect();
6704 let fix_prompt = prompt::fix_in(
6705 &self.state.instruction,
6706 &blocking_findings,
6707 this_round_verification.as_ref(),
6708 round,
6709 max_rounds,
6710 &language,
6711 &self.state.config.graph.github_language,
6712 );
6713 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6714 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6715 let (job, seat, out) = self
6716 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6717 prompt: fix_prompt.clone(),
6718 spec,
6719 seat,
6720 cwd: winner.worktree.clone(),
6721 timeout,
6722 allow_write: true,
6723 sessions,
6724 artifacts: artifacts.clone(),
6725 stem: format!("fix-{round}"),
6726 handover: None,
6727 })
6728 .await;
6729 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6732 return Ok(FixStep::Resource);
6733 }
6734 let agent_id = seat.agent.clone();
6735
6736 let mut fix = FixRecord {
6737 agent: agent_id,
6738 addressed: Vec::new(),
6739 rejected: Vec::new(),
6740 notes: String::new(),
6741 committed: false,
6742 failed: None,
6743 duration_ms: 0,
6744 continuation: None,
6745 };
6746 let mut continuation = ContinuationRecord::not_needed();
6747 let mut final_seat = seat.clone();
6748 match out {
6749 AgentOutcome::Ok(o) => {
6750 fix.duration_ms = o.duration_ms;
6751 let parsed = verdict::extract_json::<FixReport>(&o.text);
6752 let incomplete_reason = match &parsed {
6759 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6760 "the reply parsed, but it reported a command whose own CLI never \
6761 confirmed an exit status"
6762 .to_owned(),
6763 ),
6764 Ok(_) => None,
6765 Err(e) => Some(e.to_string()),
6766 };
6767 match incomplete_reason {
6768 None => {
6769 let report = parsed.expect("checked Ok above");
6770 fix.addressed = report.addressed;
6771 fix.rejected = report.rejected;
6772 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6773 }
6774 Some(reason) => {
6775 let (resumed_seat, resolved, failure, cont) = self
6776 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6777 .await;
6778 fix.duration_ms += cont.cumulative_wait_ms;
6779 continuation = cont;
6780 final_seat = resumed_seat;
6781 match resolved {
6782 Some(report) => {
6783 fix.addressed = report.addressed;
6784 fix.rejected = report.rejected;
6785 fix.notes =
6786 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6787 }
6788 None => fix.failed = failure,
6789 }
6790 }
6791 }
6792 }
6793 AgentOutcome::Dropped(o) => {
6795 fix.duration_ms = o.duration_ms;
6796 let why = o
6797 .dropped
6798 .as_ref()
6799 .map(|d| d.why.as_str())
6800 .unwrap_or("the CLI ended the stream without delivering its answer");
6801 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6802 }
6803 AgentOutcome::Quota(o) => {
6804 self.state.quota.push(QuotaLoss {
6805 seat: final_seat.key.clone(),
6806 node: "fix".to_owned(),
6807 at: Timestamp::now(),
6808 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6809 });
6810 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6811 }
6812 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6813 }
6814 fix.continuation = Some(continuation);
6815 self.state.seats.insert(final_seat.key.clone(), final_seat);
6816 let coordination = self.coordinate(&winner.worktree).await?;
6817 if let Ok(r) = git::rescue_commit(
6818 &winner.worktree,
6819 &format!("magi: review round {round} fixes (uncommitted work)"),
6820 )
6821 .await
6822 {
6823 self.state.note_withheld("fix", &r.withheld);
6824 }
6825 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6826 fix.committed = after != before;
6827 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6835 drop(coordination);
6836 let progressed = diff_after != patch;
6837 let commit_note = if fix.committed {
6838 "committed"
6839 } else {
6840 "NO new commit"
6841 };
6842 let tree_note = if progressed {
6843 "changed vs base"
6844 } else {
6845 "unchanged vs base"
6846 };
6847 self.state.event(
6848 "fix",
6849 match &fix.failed {
6850 Some(reason) => {
6856 format!(
6857 "round {round}: fixer's adoption report was lost ({reason}); \
6858 {commit_note}, tree {tree_note}"
6859 )
6860 }
6861 None => format!(
6862 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6863 {tree_note}{}",
6864 fix.addressed.len(),
6865 fix.rejected.len(),
6866 if continuation.outcome == ContinuationOutcome::Resumed {
6867 format!(
6868 " (adoption report recovered after {} continuation(s))",
6869 continuation.attempts
6870 )
6871 } else {
6872 String::new()
6873 },
6874 ),
6875 },
6876 );
6877 round_record.fix = Some(fix);
6878 round_record.progressed = progressed;
6879 Ok(FixStep::Recorded {
6880 report_lost: matches!(
6881 continuation.outcome,
6882 ContinuationOutcome::Exhausted
6883 | ContinuationOutcome::QuotaLost
6884 | ContinuationOutcome::NoSession
6885 ),
6886 })
6887 }
6888
6889 fn record_contested_handoff(&mut self) {
6923 if self.state.contested_handoff.is_some() {
6924 return;
6925 }
6926 let Some(contested) = self
6927 .state
6928 .reviews
6929 .last()
6930 .and_then(ReviewRound::contested_handoff)
6931 else {
6932 return;
6933 };
6934 self.state.event(
6935 "review",
6936 format!(
6937 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6938 wait for the owner's approval",
6939 contested.findings.len(),
6940 contested.rejecters.len()
6941 ),
6942 );
6943 self.state.contested_handoff = Some(contested);
6944 }
6945
6946 async fn stop_unresolved_review(
6947 &mut self,
6948 why: &str,
6949 shell: &[String],
6950 worktree: &Path,
6951 ) -> Result<()> {
6952 self.stop_reviewing_inner(why, shell, worktree, true).await
6953 }
6954
6955 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6956 self.stop_reviewing_inner(why, shell, worktree, false).await
6957 }
6958
6959 async fn stop_reviewing_inner(
6960 &mut self,
6961 why: &str,
6962 shell: &[String],
6963 worktree: &Path,
6964 repairable: bool,
6965 ) -> Result<()> {
6966 let round_idx = self.state.reviews.len() - 1;
6967 let needs_catchup_run = matches!(
6975 self.state.reviews[round_idx].e2e_status(),
6976 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6977 );
6978 if needs_catchup_run {
6979 let round = self.state.reviews[round_idx].round;
6980 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6981 let commands = self.state.config.verify.e2e.clone();
6982 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6983 let cache_dir = self.state.config.cache_dir();
6984 let context = format!(
6985 "round {round}: verification unresolved, catching up before the final decision"
6986 );
6987 let wait = self.repository_wait();
6988 let (outcomes, verify_retried) = with_cache_lease(
6989 &mut self.state,
6990 cache_dir.as_deref(),
6991 "e2e",
6992 "e2e",
6993 worktree,
6994 &attempted_head,
6995 timeout,
6996 &context,
6997 &wait,
6998 |state, budget| {
6999 let shell = shell.to_vec();
7000 let commands = commands.clone();
7001 let context = context.clone();
7002 async move {
7003 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7004 .await
7005 }
7006 },
7007 )
7008 .await?;
7009 let last = &mut self.state.reviews[round_idx];
7010 last.e2e = outcomes;
7011 last.verify_retried = verify_retried;
7012 last.verified_head = Some(attempted_head);
7019 last.verified_at = Some(Timestamp::now());
7020 if verify_inconclusive(&last.e2e) {
7021 self.state.save()?;
7028 return Ok(());
7029 }
7030 last.e2e_deferred = false;
7031 }
7032 let last = &self.state.reviews[round_idx];
7033 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7034
7035 match last.e2e_status() {
7036 E2eStatus::Failed => {
7037 let red: Vec<String> = last
7038 .e2e
7039 .iter()
7040 .filter(|o| !o.ok())
7041 .map(|o| {
7042 format!(
7043 "`{}` -> {:?}\n{}",
7044 o.command,
7045 o.code,
7046 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7047 )
7048 })
7049 .collect();
7050 let can_repair = last.blocking == 0 && last.fix.is_none();
7051 self.state.reviews[round_idx].clean = false;
7052 self.state
7053 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7054 self.state.status = RunStatus::Blocked;
7055 if repairable
7056 && can_repair
7057 && self.state.reviews.len() < self.state.config.graph.review_rounds
7058 {
7059 self.state.status = RunStatus::Reviewing;
7060 }
7061 }
7062 E2eStatus::ResourceBlocked => {
7067 self.state.event(
7068 "review",
7069 format!(
7070 "{why}; e2e could not run (resource unavailable); not \
7071 deciding yet"
7072 ),
7073 );
7074 }
7075 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7076 self.state.event(
7077 "review",
7078 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7079 );
7080 self.record_contested_handoff();
7081 self.state.status = RunStatus::Gating;
7082 }
7083 }
7084 self.state.save()?;
7085 Ok(())
7086 }
7087
7088 async fn gate(&mut self) -> Result<()> {
7091 if self.state.status == RunStatus::Failed
7103 || self
7104 .state
7105 .base_sync
7106 .as_ref()
7107 .is_some_and(|s| s.conflict.is_some())
7108 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7109 != Some(RunStatus::Gating)
7110 {
7111 return Ok(());
7112 }
7113 if self.state.gate_ran {
7114 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7125 self.state.status = RunStatus::Blocked;
7126 self.state.save()?;
7127 }
7128 return Ok(());
7129 }
7130 let Some(winner) = self.state.winner().cloned() else {
7131 return Ok(());
7132 };
7133 self.state.status = RunStatus::Gating;
7134 let mut outcomes = self.run_gate(&winner).await?;
7135 loop {
7136 if verify_inconclusive(&outcomes) {
7147 self.state.save()?;
7148 return Ok(());
7149 }
7150 if outcomes.iter().all(CommandOutcome::ok) {
7151 break;
7152 }
7153 match self.gate_fix_round(&winner, &outcomes).await? {
7154 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7155 GateFix::Stop => break,
7156 GateFix::Defer => {
7157 self.state.save()?;
7158 return Ok(());
7159 }
7160 }
7161 }
7162 let passed = outcomes.iter().all(CommandOutcome::ok);
7163 self.state.gate = outcomes;
7164 self.state.gate_ran = true;
7165 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7166 if !passed {
7167 self.state.status = RunStatus::Blocked;
7168 let spent = self.state.gate_fixes.len();
7169 self.state.event(
7170 "gate",
7171 if spent == 0 {
7172 "gate failed; not merging".to_owned()
7173 } else {
7174 format!("gate failed after {spent} gate-fix round(s); not merging")
7175 },
7176 );
7177 }
7178 self.state.save()?;
7179 Ok(())
7180 }
7181
7182 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7192 let commands = state.config.verify.pre_gate.clone();
7193 if commands.is_empty() {
7194 return;
7195 }
7196 let shell = state.config.shell();
7197 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7198 let (outcomes, _) = run_commands(
7199 state,
7200 "pre_gate",
7201 "pre_gate",
7202 0,
7203 &shell,
7204 &commands,
7205 &winner.worktree,
7206 timeout,
7207 )
7208 .await;
7209 for o in &outcomes {
7210 if !o.ok() {
7211 tracing::warn!(
7212 "pre_gate `{}` failed ({:?}); the gate decides",
7213 o.command,
7214 o.code
7215 );
7216 }
7217 state.event(
7218 "pre_gate",
7219 format!(
7220 "`{}` -> {}",
7221 o.command,
7222 if o.ok() {
7223 "pass".to_owned()
7224 } else {
7225 format!(
7226 "FAIL ({:?})\n{}",
7227 o.code,
7228 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7229 )
7230 }
7231 ),
7232 );
7233 }
7234 state.pre_gate = outcomes;
7235 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7236 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7237 Ok(head) => {
7238 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7239 state.pre_gate_commit = Some(head);
7240 }
7241 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7242 },
7243 Ok(false) => {}
7244 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7245 }
7246 if let Err(e) = state.save() {
7247 tracing::warn!("could not persist the pre_gate record: {e:#}");
7248 }
7249 }
7250
7251 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7254 let wait = self.repository_wait();
7255 run_landing_gate(&mut self.state, winner, &wait).await
7256 }
7257
7258 async fn gate_fix_round(
7271 &mut self,
7272 winner: &Candidate,
7273 outcomes: &[CommandOutcome],
7274 ) -> Result<GateFix> {
7275 let cap = self.state.config.graph.gate_fix_rounds;
7276 let spent = self.state.gate_fixes.len();
7277 if spent >= cap {
7278 if cap > 0 {
7279 self.state.event(
7280 "gate",
7281 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7282 );
7283 }
7284 return Ok(GateFix::Stop);
7285 }
7286 if !gate_fixable(outcomes) {
7287 self.state.event(
7288 "gate",
7289 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7290 command or similar); not spending a fix round on it",
7291 );
7292 return Ok(GateFix::Stop);
7293 }
7294 let min_free = self.state.config.disk.min_free_bytes;
7295 if min_free > 0 {
7296 match crate::disk::free_bytes(&winner.worktree) {
7297 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7298 Ok(free) => {
7299 self.state.event(
7300 "gate",
7301 format!(
7302 "only {free} bytes free ({min_free} required by `[disk] \
7303 min_free_bytes`); not spending a fix round on a failure the disk \
7304 may explain"
7305 ),
7306 );
7307 return Ok(GateFix::Stop);
7308 }
7309 Err(e) => {
7310 self.state.event(
7311 "gate",
7312 format!("free disk space could not be measured ({e:#}); no fix round"),
7313 );
7314 return Ok(GateFix::Stop);
7315 }
7316 }
7317 }
7318
7319 let attempt = spent + 1;
7320 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7321 let base = self.landing_base();
7322 let fix_prompt = prompt::gate_fix_in(
7323 &self.state.instruction,
7324 &failed,
7325 attempt,
7326 cap,
7327 &self.state.config.graph.language,
7328 &self.state.config.graph.github_language,
7329 );
7330 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7331 let sessions = self.state.config.graph.sessions;
7332 let artifacts = self.merge_repair_artifacts();
7333 self.state.event(
7334 "gate",
7335 format!("gate failed; gate-fix round {attempt} of {cap}"),
7336 );
7337 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7338 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7339 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7342 head: before.clone(),
7343 passed: false,
7344 outcomes: Vec::new(),
7345 });
7346 self.state.save()?;
7347 let (_, seat, out) = self
7348 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7349 prompt: fix_prompt.clone(),
7350 spec,
7351 seat,
7352 cwd: winner.worktree.clone(),
7353 timeout,
7354 allow_write: true,
7355 sessions,
7356 artifacts: artifacts.clone(),
7357 stem: format!("gate-fix-{attempt}"),
7358 handover: None,
7359 })
7360 .await;
7361 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7364 return Ok(GateFix::Defer);
7365 }
7366 let mut record = GateFixRecord {
7367 agent: seat.agent.clone(),
7368 failed,
7369 notes: String::new(),
7370 committed: false,
7371 error: None,
7372 };
7373 match out {
7374 AgentOutcome::Ok(o) => {
7375 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7378 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7379 }
7380 }
7381 AgentOutcome::Dropped(_) => {
7382 record.error = Some("the CLI dropped the stream".to_owned());
7383 }
7384 AgentOutcome::Quota(o) => {
7385 self.state.quota.push(QuotaLoss {
7386 seat: seat.key.clone(),
7387 node: "gate-fix".to_owned(),
7388 at: Timestamp::now(),
7389 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7390 });
7391 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7392 }
7393 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7394 }
7395 self.state.seats.insert(seat.key.clone(), seat);
7396 let coordination = self.coordinate(&winner.worktree).await?;
7397 if let Ok(r) = git::rescue_commit(
7398 &winner.worktree,
7399 &format!("magi: gate fix {attempt} (uncommitted work)"),
7400 )
7401 .await
7402 {
7403 self.state.note_withheld("gate-fix", &r.withheld);
7404 }
7405 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7406 record.committed = after != before;
7407 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7408 let note = record.error.clone();
7409 self.state.gate_fixes.push(record);
7410 self.state.save()?;
7411 if !changed {
7412 self.state.event(
7413 "gate",
7414 match note {
7415 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7416 None => format!("gate-fix round {attempt}: the tree did not change"),
7417 },
7418 );
7419 return Ok(GateFix::Stop);
7420 }
7421 self.state.event(
7422 "gate",
7423 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7424 );
7425
7426 drop(coordination);
7427 let wait = self.repository_wait();
7428 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7429 }
7430
7431 async fn guarded_pr_message(
7438 &mut self,
7439 winner: &Candidate,
7440 facts: Option<&BranchFacts>,
7441 posting: bool,
7442 ) -> Result<Option<PrMessage>> {
7443 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7444 let head = match final_head.as_deref() {
7445 Some(h) => crate::pr_outcome::FinalHead::At(h),
7446 None => crate::pr_outcome::FinalHead::Unknown,
7447 };
7448 let summarized = if posting {
7450 Some(self.summarize_change(winner, final_head.as_deref()).await)
7451 } else {
7452 None
7453 };
7454 if self.state.resource_block.is_some() {
7459 self.state.event(
7460 "pr-summary",
7461 "a shared resource was busy; nothing was published, resume to summarize again",
7462 );
7463 return Ok(None);
7464 }
7465 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7466 anyhow::bail!(
7467 "the final commit moved during public description generation; retry before publishing"
7468 );
7469 }
7470 let note = match (&summarized, &self.state.change_summary) {
7471 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7472 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7473 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7474 (None, _) => ChangeNote::NotAsked,
7475 };
7476 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7477 if !posting {
7478 let identity = crate::scrub::Identity::current();
7479 return Ok(Some(PrMessage {
7480 title: crate::scrub::scrub(&pr.title, &identity),
7481 body: crate::scrub::scrub(&pr.body, &identity),
7482 }));
7483 }
7484 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7487 if let Some(known) = self
7488 .state
7489 .github_text
7490 .clone()
7491 .filter(|g| g.fingerprint == raw_fp)
7492 {
7493 return self.settle_withheld(winner, &pr, known).await;
7494 }
7495 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7496 let mut violations =
7497 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7498 let identity = crate::scrub::Identity::current();
7499 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7500 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7501 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7502 {
7503 violations.push(crate::github_text::Violation::SensitiveData);
7504 }
7505 let has_language = violations
7509 .iter()
7510 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7511 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7512 self.state.event(
7513 "github-text",
7514 if has_language {
7515 format!("description rejected: {violations:?}; requesting one rewrite")
7516 } else {
7517 format!("description rejected: {violations:?}; asking the owner")
7518 },
7519 );
7520 let mut rewritten = false;
7521 let mut rejected_rewrite = None;
7522 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7523 {
7524 let output = text;
7525 if let Some(title) = summary_title(&output) {
7526 let raw_body = summary_without_title(&output);
7529 let clean_title = crate::scrub::scrub(&title, &identity);
7530 let mut body = crate::scrub::scrub(&raw_body, &identity);
7531 let redacted = clean_title != title || body != raw_body;
7532 let title = clean_title;
7533 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7534 if !complete {
7535 self.state.event(
7536 "github-text",
7537 "description rewrite omitted required findings",
7538 );
7539 }
7540 if !body.trim().is_empty() {
7541 body.push_str("\n\n");
7542 body.push_str(&public_pr_evidence(&self.state, head));
7543 }
7544 let marker = format!("magi:run/{}", self.state.id);
7545 if !body.trim().is_empty() && !body.contains(&marker) {
7546 body.push_str(&format!("\n\n{marker}"));
7547 }
7548 let rewrite = PrMessage {
7549 title: title.clone(),
7550 body: body.clone(),
7551 };
7552 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7553 let mut failed = crate::github_text::check_for(
7554 &self.state.config.graph,
7555 &title,
7556 &body,
7557 redecision,
7558 );
7559 if (!crate::github_text::shareable(&title)
7560 || !crate::github_text::shareable(&body))
7561 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7562 {
7563 failed.push(crate::github_text::Violation::SensitiveData);
7564 }
7565 if complete
7566 && !title.trim().is_empty()
7567 && !body.trim().is_empty()
7568 && !failed.is_empty()
7569 {
7570 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7571 title: title.clone(),
7572 body: body.clone(),
7573 failed: failed
7574 .iter()
7575 .map(|v| crate::github_text::category(*v).to_owned())
7576 .collect(),
7577 });
7578 }
7579 if complete
7580 && !title.trim().is_empty()
7581 && !raw_body.trim().is_empty()
7582 && failed.is_empty()
7583 {
7584 decision = redecision;
7585 pr = PrMessage { title, body };
7586 rewritten = true;
7587 self.state
7588 .event("github-text", "description rewrite passed");
7589 if redacted {
7590 self.state.event(
7591 "github-text",
7592 "sensitive data removed from description rewrite",
7593 );
7594 }
7595 }
7596 }
7597 }
7598 if !rewritten {
7599 self.state.event(
7600 "github-text",
7601 "description rewrite unavailable or rejected; asking the owner",
7602 );
7603 if let Some(w) =
7604 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7605 {
7606 self.retire_github_text_question("the withheld text changed");
7609 let record = crate::run::GithubTextAsk {
7610 rewrite: rejected_rewrite,
7611 fingerprint: raw_fp,
7612 title: w.title,
7613 body: w.body,
7614 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7615 question: None,
7616 asks: 0,
7617 resolved: false,
7618 chosen_title: None,
7619 choice: None,
7620 };
7621 return self.settle_withheld(winner, &pr, record).await;
7622 }
7623 }
7624 }
7625 let (title, body) =
7626 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7627 Ok(Some(PrMessage { title, body }))
7628 }
7629
7630 async fn summarize_change(
7641 &mut self,
7642 winner: &Candidate,
7643 head: Option<&str>,
7644 ) -> std::result::Result<(), String> {
7645 let Some(head) = head else {
7646 return Err("the final commit could not be read".to_owned());
7647 };
7648 let remote = self.state.config.merge.remote.clone();
7649 let tracking = format!("{remote}/{}", self.state.base_branch);
7650 if !git::rev_exists(&winner.worktree, &tracking).await {
7651 return Err(format!("the remote base `{tracking}` could not be read"));
7652 }
7653 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7654 Ok(b) if !b.is_empty() => b,
7655 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7656 };
7657 let evidence = crate::pr_outcome::summary_evidence(
7658 &self.state,
7659 crate::pr_outcome::FinalHead::At(head),
7660 );
7661 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7662 if self
7663 .state
7664 .change_summary
7665 .as_ref()
7666 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7667 {
7668 return Ok(());
7669 }
7670 if self.state.change_summary.take().is_some() {
7671 self.state.event(
7672 "pr-summary",
7673 "the final diff or evidence changed; the earlier change summary was discarded",
7674 );
7675 }
7676 if let Err(e) = write_artifact(
7678 &self.state,
7679 &format!("public-description-{fingerprint}.md"),
7680 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7681 ) {
7682 tracing::warn!("could not save public description evidence: {e:#}");
7683 }
7684 let chain = match agent::pick_chain(
7685 &self.state.config.agents,
7686 self.state.config.roles.synthesizer.as_ref(),
7687 &agent::installed,
7688 "synthesizer",
7689 ) {
7690 Ok(chain) => chain,
7691 Err(e) => {
7692 tracing::warn!("no agent for the change summary: {e:#}");
7693 self.state.event("pr-summary", "no summary agent available");
7694 return Err("no agent was available to summarize it".to_owned());
7695 }
7696 };
7697 let subjects = git::subjects(&winner.worktree, &base, head)
7698 .await
7699 .unwrap_or_default();
7700 let stat = git::diff_stat(&winner.worktree, &base, head)
7701 .await
7702 .unwrap_or_default();
7703 let stat: String = stat.chars().take(6_000).collect();
7704 let mut prompt = crate::prompt::change_summary_in(
7705 &base,
7706 head,
7707 &subjects,
7708 stat.trim(),
7709 &self.state.config.graph.github_language,
7710 );
7711 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7712 let sessions = self.state.config.graph.sessions;
7713 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7714 let prompts = self.state.config.prompts.clone();
7715 let cache = self.state.config.cache_dir();
7716 let run = self.state.id.clone();
7717 for (i, spec) in chain.into_iter().enumerate() {
7718 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7719 let github_spec = match agent::for_github_text(
7720 &spec,
7721 &self.state.config.graph.github_language,
7722 &winner.worktree,
7723 ) {
7724 Ok(spec) => spec,
7725 Err(e) => {
7726 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7727 continue;
7728 }
7729 };
7730 let job = SeatJob {
7731 spec: github_spec,
7732 seat,
7733 cwd: winner.worktree.clone(),
7734 prompt: prompt.clone(),
7735 timeout,
7736 allow_write: false,
7737 sessions,
7738 artifacts: agent::artifacts_dir(&self.state.dir()),
7739 stem: if i == 0 {
7740 "change-summary".to_owned()
7741 } else {
7742 format!("change-summary-{}", spec.id)
7743 },
7744 handover: None,
7745 };
7746 let ctx = WaveCtx {
7747 run: &run,
7748 node: "pr-summary",
7749 prompts: &prompts,
7750 cache: cache.as_deref(),
7751 round: None,
7752 carry_seats: false,
7753 pauses: Some((&self.pause, &self.interrupt)),
7754 };
7755 let (seat, outcome) =
7756 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7757 self.state.seats.insert(seat.key.clone(), seat);
7758 let text = match outcome {
7759 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7760 AgentOutcome::ResourceBlocked(why) => {
7761 if self.state.resource_block.is_none() {
7764 self.state.resource_block = Some(crate::run::ResourceBlock {
7765 cause: crate::run::ResourceBlockCause::Other,
7766 node: "pr-summary".to_owned(),
7767 kind: "shared resource".to_owned(),
7768 detail: why,
7769 at: Timestamp::now(),
7770 });
7771 }
7772 return Err("a shared resource was busy".to_owned());
7773 }
7774 _ => continue,
7775 };
7776 let blind = &self.state.config.blind;
7777 let Some(title) = summary_title(&text) else {
7778 continue;
7779 };
7780 let body: String = summary_without_title(&text)
7781 .lines()
7782 .filter(|l| !l.contains("magi:run/"))
7783 .collect::<Vec<_>>()
7784 .join("\n");
7785 let body = blind::sanitize_prose(body.trim(), blind);
7786 let title = blind::sanitize_prose(&title, blind);
7787 if body.trim().is_empty() || title.trim().is_empty() {
7788 continue;
7789 }
7790 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7791 continue;
7792 }
7793 if git::rev_parse(&winner.worktree, "HEAD")
7795 .await
7796 .ok()
7797 .as_deref()
7798 != Some(head)
7799 {
7800 return Err("the branch moved while the summary was written".to_owned());
7801 }
7802 if git::merge_base(&winner.worktree, &tracking, head)
7803 .await
7804 .ok()
7805 .as_deref()
7806 != Some(base.as_str())
7807 || crate::pr_outcome::summary_evidence(
7808 &self.state,
7809 crate::pr_outcome::FinalHead::At(head),
7810 ) != evidence
7811 {
7812 return Err("the summary evidence changed while it was written".to_owned());
7813 }
7814 self.state.change_summary = Some(crate::run::ChangeSummary {
7815 head: head.to_owned(),
7816 base,
7817 evidence: fingerprint,
7818 title,
7819 body,
7820 });
7821 self.state.event(
7822 "pr-summary",
7823 format!("change summary written by {} for the final diff", spec.id),
7824 );
7825 if let Err(e) = self.state.save() {
7826 tracing::warn!("could not save the change summary: {e:#}");
7827 }
7828 return Ok(());
7829 }
7830 self.state
7831 .event("pr-summary", "no agent produced a change summary");
7832 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7833 }
7834
7835 async fn rewrite_pr_text(
7845 &mut self,
7846 winner: &Candidate,
7847 pr: &PrMessage,
7848 violations: &[crate::github_text::Violation],
7849 ) -> Option<String> {
7850 let prompt = format!(
7851 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7852 pr.title, pr.body
7853 );
7854 let sessions = self.state.config.graph.sessions;
7855 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7856 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7859 let key = format!("impl-{}", winner.label);
7860 if let Some(spec) = self
7861 .state
7862 .config
7863 .agents
7864 .iter()
7865 .find(|a| a.id == winner.agent)
7866 .cloned()
7867 && let Some(existing) = self.state.seats.get(&key)
7868 && existing.agent == spec.id
7869 && agent::has_session(spec.kind, existing, sessions)
7870 {
7871 attempts.push((spec, Some(existing.clone())));
7872 }
7873 match agent::pick_chain(
7874 &self.state.config.agents,
7875 self.state.config.roles.language_judge.as_ref(),
7876 &agent::installed,
7877 "language_judge",
7878 ) {
7879 Ok(chain) => {
7880 for spec in chain {
7881 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7882 continue;
7883 }
7884 attempts.push((spec, None));
7885 }
7886 }
7887 Err(e) if attempts.is_empty() => {
7888 tracing::warn!("no agent for the description rewrite: {e:#}");
7889 self.state
7890 .event("github-text", "no rewrite agent available");
7891 return None;
7892 }
7893 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7894 }
7895 let prompts = self.state.config.prompts.clone();
7896 let cache = self.state.config.cache_dir();
7897 let run = self.state.id.clone();
7898 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7899 let resumed = resumed_seat.is_some();
7900 let seat = resumed_seat.unwrap_or_else(|| {
7901 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7902 });
7903 let github_spec = match agent::for_github_text(
7904 &spec,
7905 &self.state.config.graph.github_language,
7906 &winner.worktree,
7907 ) {
7908 Ok(spec) => spec,
7909 Err(e) => {
7910 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7911 continue;
7912 }
7913 };
7914 let job = SeatJob {
7915 spec: github_spec,
7916 seat,
7917 cwd: winner.worktree.clone(),
7918 prompt: prompt.clone(),
7919 timeout: retry_budget(timeout_full, resumed),
7920 allow_write: false,
7921 sessions,
7922 artifacts: agent::artifacts_dir(&self.state.dir()),
7923 stem: if i == 0 {
7924 "github-text-rewrite".to_owned()
7925 } else {
7926 format!("github-text-rewrite-{}", spec.id)
7927 },
7928 handover: None,
7929 };
7930 let ctx = WaveCtx {
7931 run: &run,
7932 node: "github-text",
7933 prompts: &prompts,
7934 cache: cache.as_deref(),
7935 round: None,
7936 carry_seats: false,
7937 pauses: Some((&self.pause, &self.interrupt)),
7938 };
7939 let (seat, outcome) =
7940 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7941 self.state.seats.insert(seat.key.clone(), seat);
7942 match outcome {
7943 AgentOutcome::Ok(o)
7944 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7945 {
7946 return Some(o.text);
7947 }
7948 AgentOutcome::ResourceBlocked(_) => return None,
7949 _ => {}
7950 }
7951 }
7952 None
7953 }
7954
7955 fn retire_github_text_question(&mut self, why: &str) {
7957 let Some(id) = self
7958 .state
7959 .github_text
7960 .as_ref()
7961 .and_then(|g| g.question.clone())
7962 else {
7963 return;
7964 };
7965 let _ = ask::Questions::open().update(&id, |q| {
7966 q.abandon(why);
7967 Ok(())
7968 });
7969 }
7970
7971 async fn settle_withheld(
7975 &mut self,
7976 winner: &Candidate,
7977 pr: &PrMessage,
7978 mut g: crate::run::GithubTextAsk,
7979 ) -> Result<Option<PrMessage>> {
7980 use crate::github_text as gt;
7981 let store = ask::Questions::open();
7982 let timeout = self.state.config.graph.answer_timeout;
7983 if !g.resolved {
7984 let mut standing = g
7985 .question
7986 .as_ref()
7987 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7988 if let Some(q) = standing.as_ref()
7989 && q.status.open()
7990 && gt::expired(q, timeout)
7991 {
7992 let why = format!("no answer within {}s of asking", timeout.max(1));
7993 standing = store
7994 .update(&q.id, |q| {
7995 q.abandon(&why);
7996 Ok(())
7997 })
7998 .ok()
7999 .map(|(q, ())| q);
8000 }
8001 let mut ask_again: Option<Option<Vec<String>>> = None;
8002 match standing {
8003 None if g.asks <= 1 => {
8006 g.asks = 0;
8007 ask_again = Some(None);
8008 }
8009 None => {
8010 g.resolved = true;
8011 g.chosen_title = None;
8012 g.choice = Some(GithubTextChoice::Fallback);
8013 }
8014 Some(q) if q.status.open() => {
8015 self.park_on_github_text(&g)?;
8016 return Ok(None);
8017 }
8018 Some(q) => match gt::read_reply(&q) {
8019 gt::Reply::Fallback => {
8020 g.resolved = true;
8021 g.chosen_title = None;
8022 g.choice = Some(GithubTextChoice::Fallback);
8023 }
8024 gt::Reply::Original => {
8025 g.resolved = true;
8026 g.chosen_title = None;
8027 g.choice = Some(GithubTextChoice::Original);
8028 }
8029 gt::Reply::Rewrite => {
8030 g.resolved = true;
8031 g.chosen_title = None;
8032 g.choice = Some(if g.rewrite.is_some() {
8033 GithubTextChoice::Rewrite
8034 } else {
8035 GithubTextChoice::Fallback
8036 });
8037 }
8038 gt::Reply::Title(text) => {
8039 let first = text
8040 .lines()
8041 .map(str::trim)
8042 .find(|l| !l.is_empty())
8043 .unwrap_or("");
8044 let decision = if first.is_empty() {
8045 None
8046 } else {
8047 let candidate = PrMessage {
8048 title: first.to_owned(),
8049 body: String::new(),
8050 };
8051 self.judge_pr_language(&winner.worktree, &candidate).await
8052 };
8053 match gt::vet_title(
8054 &text,
8055 decision,
8056 self.state.config.graph.github_english(),
8057 ) {
8058 Ok(title) => {
8059 g.resolved = true;
8060 g.chosen_title = Some(title);
8061 g.choice = Some(GithubTextChoice::MyTitle);
8062 }
8063 Err(cats) if g.asks < 2 => {
8064 ask_again = Some(Some(
8065 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8066 ))
8067 }
8068 Err(_) => {
8069 g.resolved = true;
8070 g.chosen_title = None;
8071 g.choice = Some(GithubTextChoice::Fallback);
8072 }
8073 }
8074 }
8075 },
8076 }
8077 if let Some(rejected) = ask_again {
8078 let w = gt::Withheld {
8081 title: g.title,
8082 body: g.body,
8083 categories: g.categories.clone(),
8084 };
8085 let artifact = match crate::run::write_artifact(
8088 &self.state,
8089 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8090 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8091 ) {
8092 Ok(p) => Some(p),
8093 Err(e) => {
8094 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8095 None
8096 }
8097 };
8098 let (detail, comparison) = gt::question_content(
8099 &w,
8100 &pr.title,
8101 &pr.body,
8102 &self.state.id,
8103 rejected.as_deref(),
8104 artifact.as_deref(),
8105 g.rewrite.as_ref(),
8106 );
8107 let mut q = ask::Question::new(
8108 self.state.id.clone(),
8109 gt::ASK_NODE.to_owned(),
8110 gt::ASK_SEAT.to_owned(),
8111 gt::question_summary(&self.state.config.graph.language, &w),
8112 detail,
8113 gt::question_choices(&w, g.rewrite.as_ref()),
8114 );
8115 q.posting_comparison = Some(comparison);
8116 q.answer_timeout = timeout;
8117 g.question = Some(q.id.clone());
8120 g.asks += 1;
8121 self.state.github_text = Some(g.clone());
8122 self.state.event(
8123 "github-text",
8124 format!("asking the owner about the withheld text ({})", q.short()),
8125 );
8126 self.state.save()?;
8127 store
8128 .put_executable(&mut q)
8129 .context("file the posting-gate question")?;
8130 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8131 tracing::warn!(
8132 "could not notify about posting-gate question {}: {e:#}",
8133 q.short()
8134 );
8135 }
8136 self.park_on_github_text(&g)?;
8137 return Ok(None);
8138 }
8139 }
8140 let resolved = g.chosen_title.clone();
8141 let original = g.choice == Some(GithubTextChoice::Original);
8142 self.state.event(
8144 "github-text",
8145 if original {
8146 format!(
8147 "posting the original text for: {}",
8148 match (g.title, g.body) {
8149 (true, true) => "title, description",
8150 (true, false) => "title",
8151 _ => "description",
8152 }
8153 )
8154 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8155 "posting the saved rewrite title and description".to_owned()
8156 } else if resolved.is_some() {
8157 "posting the owner's replacement title".to_owned()
8158 } else {
8159 "posting the neutral text for the withheld fields".to_owned()
8160 },
8161 );
8162 self.state.github_text = Some(g.clone());
8163 self.state.save()?;
8164 if g.choice == Some(GithubTextChoice::Rewrite)
8165 && let Some(r) = g.rewrite.as_ref()
8166 {
8167 return Ok(Some(PrMessage {
8168 title: r.title.clone(),
8169 body: r.body.clone(),
8170 }));
8171 }
8172 let w = gt::Withheld {
8173 title: g.title,
8174 body: g.body,
8175 categories: g.categories.clone(),
8176 };
8177 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8178 if let Some(t) = resolved {
8179 title = t;
8180 }
8181 let english = gt::LanguageDecision {
8183 title_english: true,
8184 body_english: true,
8185 body_complete: true,
8186 };
8187 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8188 if original {
8189 if g.title {
8192 title = pr.title.clone();
8193 }
8194 if g.body {
8195 body = pr.body.clone();
8196 }
8197 }
8198 Ok(Some(PrMessage { title, body }))
8199 }
8200
8201 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8202 self.state.github_text = Some(g.clone());
8203 self.state.end_pass_parked(ParkCause::Question);
8204 self.state.parked = true;
8205 self.state.event(
8206 "github-text",
8207 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8208 );
8209 self.state.save()
8210 }
8211
8212 async fn judge_pr_language(
8215 &mut self,
8216 cwd: &Path,
8217 pr: &PrMessage,
8218 ) -> Option<crate::github_text::LanguageDecision> {
8219 self.state.config.roles.language_judge.as_ref()?;
8220 if !self.state.config.graph.language_gate() {
8221 return None;
8222 }
8223 let decision =
8224 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8225 self.state.event(
8226 "github-text",
8227 if decision.is_some() {
8228 "language decided by the language judge"
8229 } else {
8230 "language judge unavailable; using built-in heuristics"
8231 },
8232 );
8233 decision
8234 }
8235
8236 async fn merge(&mut self) -> Result<()> {
8237 if self
8252 .state
8253 .base_sync
8254 .as_ref()
8255 .is_some_and(|s| s.conflict.is_some())
8256 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8257 != Some(RunStatus::Gating)
8258 || !self.state.gate_allows_progress()
8266 {
8267 return Ok(());
8268 }
8269 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8278 return Ok(());
8279 }
8280 let Some(winner) = self.state.winner().cloned() else {
8281 return Ok(());
8282 };
8283 let repo = self.state.repo.clone();
8284 let base = self.state.base_branch.clone();
8285 let mode = self.state.config.merge.mode;
8286 let style = self.state.config.merge.style;
8287 let facts = if is_review_run(&self.state) {
8288 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8289 let start = review_base(
8290 &repo,
8291 &self.state.config.merge.remote,
8292 &base,
8293 &self.state.base_commit,
8294 &winner.branch,
8295 )
8296 .await;
8297 branch_facts(&repo, &start, &winner.branch).await
8298 } else {
8299 None
8300 };
8301 let leaked = if is_review_run(&self.state) {
8304 leaked_subjects(&self.state, &winner.branch).await
8305 } else {
8306 Some(Vec::new())
8307 };
8308 let Some(pr) = self
8312 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8313 .await?
8314 else {
8315 return Ok(());
8316 };
8317 let message = pr.commit_message();
8318
8319 let outcome = match mode {
8320 MergeMode::None => MergeOutcome {
8321 mode,
8322 ok: true,
8323 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8324 empty: false,
8325 },
8326 MergeMode::Pr | MergeMode::Local
8327 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8328 {
8329 MergeOutcome {
8330 mode,
8331 ok: false,
8332 detail: empty_candidate_detail(&self.state, &base),
8333 empty: true,
8334 }
8335 }
8336 MergeMode::Local => {
8337 let on = git::current_branch(&repo).await?;
8338 if on.as_deref() != Some(base.as_str()) {
8339 MergeOutcome {
8340 mode,
8341 ok: false,
8342 detail: format!(
8343 "{} has {} checked out, not the base branch {base}",
8344 repo.display(),
8345 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8346 ),
8347 empty: false,
8348 }
8349 } else if !git::is_clean(&repo).await? {
8350 MergeOutcome {
8351 mode,
8352 ok: false,
8353 detail: format!("{} is dirty; refusing to merge", repo.display()),
8354 empty: false,
8355 }
8356 } else {
8357 let out = match style {
8358 MergeStyle::Merge => {
8359 git::merge_no_ff(&repo, &winner.branch, &message).await?
8360 }
8361 MergeStyle::Squash => {
8362 git::merge_squash(&repo, &winner.branch, &message).await?
8363 }
8364 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8365 };
8366 MergeOutcome {
8367 mode,
8368 ok: out.ok(),
8369 detail: if out.ok() { out.stdout } else { out.stderr },
8370 empty: false,
8371 }
8372 }
8373 }
8374 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8375 self.state.pr_publication_pending = true;
8376 self.state.save()?;
8377 let out =
8378 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8379 self.state.pr_publication_pending = out.is_err();
8380 MergeOutcome {
8381 mode,
8382 ok: out.is_ok(),
8383 detail: match out {
8384 Ok(url) => url,
8385 Err(e) => format!("{e:#}"),
8386 },
8387 empty: false,
8388 }
8389 }
8390 MergeMode::Pr => {
8391 let remote = self.state.config.merge.remote.clone();
8392 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8393 if !pushed.ok() {
8394 MergeOutcome {
8395 mode,
8396 ok: false,
8397 detail: pushed.stderr,
8398 empty: false,
8399 }
8400 } else {
8401 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8407 let out = match pr_merge_plan(found) {
8408 PrPlan::Create => {
8409 gh_pr_create(
8410 &winner.worktree,
8411 &base,
8412 &winner.branch,
8413 &pr.title,
8414 &pr.body,
8415 )
8416 .await
8417 }
8418 PrPlan::Adopt { url, title } => {
8419 self.state
8420 .event("merge", format!("Pr: adopted open pull request {url}"));
8421 if self.state.github_text.as_ref().is_some_and(|g| {
8422 g.resolved
8423 && g.choice == Some(GithubTextChoice::Rewrite)
8424 && g.rewrite.is_some()
8425 }) {
8426 land::set_pr_text_verbatim(
8429 &winner.worktree,
8430 &url,
8431 &pr.title,
8432 &pr.body,
8433 )
8434 .await
8435 .map(|()| url)
8436 } else {
8437 if title != pr.title
8438 && (!is_review_run(&self.state)
8439 || leaked
8440 .as_deref()
8441 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8442 && let Err(e) =
8443 if self.state.github_text.as_ref().is_some_and(|g| {
8444 g.resolved
8445 && g.title
8446 && g.choice == Some(GithubTextChoice::Original)
8447 }) {
8448 land::set_pr_title_verbatim(
8450 &winner.worktree,
8451 &url,
8452 &pr.title,
8453 )
8454 .await
8455 } else {
8456 land::set_pr_title(
8457 &mut self.state,
8458 &winner.worktree,
8459 &url,
8460 &pr.title,
8461 )
8462 .await
8463 }
8464 {
8465 tracing::warn!("could not refresh title of {url}: {e:#}");
8466 self.state
8467 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8468 }
8469 Ok(url)
8470 }
8471 }
8472 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8473 };
8474 match out {
8475 Ok(url) => MergeOutcome {
8476 mode,
8477 ok: true,
8478 detail: url,
8479 empty: false,
8480 },
8481 Err(e) => MergeOutcome {
8482 mode,
8483 ok: false,
8484 detail: e.to_string(),
8485 empty: false,
8486 },
8487 }
8488 }
8489 }
8490 };
8491
8492 self.state.status = match (mode, outcome.ok) {
8493 (MergeMode::None, _) => RunStatus::Ready,
8494 (_, true) if outcome.empty => RunStatus::Ready,
8495 (MergeMode::Pr, true) => RunStatus::Ready,
8496 (_, true) => RunStatus::Merged,
8497 (_, false) => RunStatus::Blocked,
8498 };
8499 self.state.event(
8500 "merge",
8501 format!(
8502 "{:?}: {}",
8503 mode,
8504 outcome.detail.lines().next().unwrap_or("")
8505 ),
8506 );
8507 self.state.merge = Some(outcome);
8508 self.state.save()?;
8509
8510 if self.state.config.graph.land
8516 && mode == MergeMode::Pr
8517 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8518 {
8519 self.run_land().await?;
8520 }
8521 self.settle_questions();
8526 Ok(())
8527 }
8528
8529 async fn run_land(&mut self) -> Result<()> {
8540 crate::pr_head::ensure_owner(&self.state)?;
8541 let url = self
8542 .state
8543 .pr
8544 .as_ref()
8545 .map(|pr| pr.url.clone())
8546 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8547 .unwrap_or_default();
8548 let url = url.lines().next().unwrap_or("").trim().to_owned();
8549 if !url.starts_with("http") {
8550 return Ok(());
8551 }
8552 loop {
8555 match land::land(&mut self.state, &url).await {
8556 Ok(_)
8557 if self
8558 .state
8559 .merge_repairs
8560 .last()
8561 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8562 {
8563 if self.repair_merge_review().await? {
8564 continue;
8565 }
8566 }
8567 Ok(pr) if self.state.parked => {
8568 let _ = pr;
8572 }
8573 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8574 self.state.status = RunStatus::Superseded;
8575 self.state.save()?;
8576 }
8577 Ok(pr) => {
8578 self.state.status = match pr.state {
8579 land::PrLifecycle::Merged => RunStatus::Merged,
8580 _ => RunStatus::Blocked,
8581 };
8582 if bump::should_release_bump(self.state.status)
8589 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8590 {
8591 self.state
8597 .event("bump", format!("release bump skipped: {e:#}"));
8598 }
8599 if self.state.status == RunStatus::Merged {
8602 crate::followup::after_merge(&mut self.state, &pr.url).await;
8603 }
8604 self.state.save()?;
8605 }
8606 Err(e) => {
8607 if e.is::<crate::repo_lock::ResourceBlocked>() {
8608 return Err(e);
8609 }
8610 self.state.status = RunStatus::Blocked;
8611 self.state.event("land", format!("gave up: {e}"));
8612 self.state.save()?;
8613 }
8614 }
8615 break;
8616 }
8617 Ok(())
8618 }
8619
8620 async fn repair_merge_review(&mut self) -> Result<bool> {
8623 use crate::run::MergeRepairPhase as Phase;
8624 crate::pr_head::ensure_owner(&self.state)?;
8625 let i = self.state.merge_repairs.len() - 1;
8626 let request = self.state.merge_repairs[i].clone();
8627 if request.phase == Phase::Fixing {
8628 let restored = match self.state.winner().cloned() {
8629 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8630 None => Err(anyhow::anyhow!("repair has no winning branch")),
8631 };
8632 let why = match restored {
8633 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8634 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8635 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8636 };
8637 self.hold_merge_repair(i, why)?;
8638 return Ok(false);
8639 }
8640 let failure = if i >= self.state.config.graph.land_rounds
8641 || self.state.config.graph.review_rounds == 0
8642 {
8643 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8644 } else {
8645 None
8646 };
8647 if let Some(why) = failure {
8648 self.hold_merge_repair(i, why)?;
8649 return Ok(false);
8650 }
8651 if request.phase == Phase::Requested {
8652 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8653 Ok(pr) => pr,
8654 Err(e) => {
8655 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8656 return Ok(false);
8657 }
8658 };
8659 self.state.merge_repairs[i].phase = Phase::Fixing;
8660 self.state.land_approval = None;
8661 self.state.land_armed_head = None;
8662 self.state.save()?;
8663 let evidence = serde_json::to_string_pretty(&request)?;
8666 let reason = format!(
8667 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8668 );
8669 let budget = self.state.config.graph.land_rounds;
8670 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8671 match result {
8672 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8673 Ok(land::Fixed::Failed(why)) => {
8674 self.hold_merge_repair(
8675 i,
8676 format!(
8677 "repair failed: {why}; inspect the branch and run an explicit review"
8678 ),
8679 )?;
8680 return Ok(false);
8681 }
8682 Err(e) => {
8683 self.hold_merge_repair(
8684 i,
8685 format!(
8686 "repair failed: {e:#}; inspect the branch and run an explicit review"
8687 ),
8688 )?;
8689 return Ok(false);
8690 }
8691 }
8692 self.begin_merge_repair_review(i)?;
8696 }
8697 self.review_loop().await?;
8698 self.gate().await?;
8699 if self.resource_halt()? || self.state.parked {
8700 return Ok(false);
8701 }
8702 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8703 != Some(RunStatus::Gating)
8704 || !self.state.gate_allows_progress()
8705 {
8706 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8707 return Ok(false);
8708 }
8709 let winner = self
8712 .state
8713 .winner()
8714 .cloned()
8715 .context("repair has no winning branch")?;
8716 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8717 if self.state.gate_head.as_deref() != Some(after.as_str())
8718 || !git::is_clean(&winner.worktree).await?
8719 {
8720 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8721 return Ok(false);
8722 }
8723 let expected = self
8724 .state
8725 .pr_pushed_head
8726 .as_deref()
8727 .unwrap_or(&request.head)
8728 .to_owned();
8729 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8730 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8733 let refspec = format!("{after}:refs/heads/{public}");
8734 let push = git::git_raw(
8735 &winner.worktree,
8736 &["push", &lease, &self.state.config.merge.remote, &refspec],
8737 )
8738 .await;
8739 match push {
8740 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8741 Ok(out) => {
8742 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8743 return Ok(false);
8744 }
8745 Err(e) => {
8746 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8747 return Ok(false);
8748 }
8749 }
8750 self.state.merge_repairs[i].phase = Phase::Complete;
8751 self.state.status = RunStatus::Landing;
8752 self.state.event(
8753 "repair",
8754 "new review and gate completed; fresh owner merge decision required",
8755 );
8756 self.state.save()?;
8757 Ok(true)
8758 }
8759
8760 fn merge_repair_artifacts(&self) -> PathBuf {
8761 let artifacts = agent::artifacts_dir(&self.state.dir());
8762 match self
8763 .state
8764 .merge_repairs
8765 .last()
8766 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8767 {
8768 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8769 None => artifacts,
8770 }
8771 }
8772
8773 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8774 self.state.reviews.clear();
8775 self.state.gate.clear();
8776 self.state.gate_ran = false;
8777 self.state.gate_head = None;
8778 self.state.gate_fixes.clear();
8779 self.state.gate_repair_e2e = None;
8780 self.state.pre_gate.clear();
8781 self.state.pre_gate_commit = None;
8782 self.state.contested_handoff = None;
8783 self.state.land_reruns = None;
8784 self.state.land_approval = None;
8785 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8786 self.state.status = RunStatus::Reviewing;
8787 self.state.save()
8788 }
8789
8790 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8791 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8792 self.state.merge_repairs[index].failure = Some(why.clone());
8793 self.state.status = RunStatus::Blocked;
8794 self.state.event("repair", why);
8795 self.state.save()
8796 }
8797
8798 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8802 if let Some(existing) = self.state.seats.get(key)
8803 && existing.agent == agent
8804 {
8805 return existing.clone();
8806 }
8807 let fresh = if self.state.seats.contains_key(key) {
8810 handover_seat(key, agent, self.state.next_seat_seed())
8811 } else {
8812 SeatState::new(key, agent, self.state.seed)
8813 };
8814 self.state.seats.insert(key.to_owned(), fresh.clone());
8815 fresh
8816 }
8817
8818 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8824 match self.state.seats.get(key) {
8825 Some(s) if s.agent != spec.id => {
8826 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8827 }
8828 _ => spec,
8829 }
8830 }
8831
8832 fn view(&self, c: &Candidate) -> CandidateView {
8834 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8835 .unwrap_or_default();
8836 let (patch, _) = blind::sanitize_patch(
8837 &format!("candidate {} patch", c.label),
8838 &raw,
8839 &self.state.config.blind,
8840 );
8841 CandidateView {
8842 label: c.label,
8843 branch: c.branch.clone(),
8844 summary: c.summary.clone(),
8845 stat: c.stat.clone(),
8846 patch,
8847 }
8848 }
8849
8850 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8852 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8853 prompt::judge(
8854 "(see above)",
8855 &views,
8856 self.roles.judges.len(),
8857 base_short,
8858 "en",
8859 )
8860 }
8861
8862 fn vote_prompt_full(
8868 &self,
8869 j: usize,
8870 labels: &[char],
8871 language: &str,
8872 candidates: &[Candidate],
8873 base_short: &str,
8874 ) -> String {
8875 let mut text = format!(
8876 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8877 prompt::final_vote(labels, language),
8878 self.state.instruction,
8879 self.candidate_block(candidates, base_short)
8880 );
8881 if let Some(own) = self
8882 .state
8883 .judgements
8884 .get(j)
8885 .filter(|r| !r.ranking.is_empty())
8886 {
8887 let reasons = own
8888 .reasons
8889 .iter()
8890 .map(|(k, v)| format!("- {k}: {v}"))
8891 .collect::<Vec<_>>()
8892 .join("\n");
8893 text.push_str(&format!(
8894 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8895 own.ranking.iter().collect::<String>(),
8896 if reasons.is_empty() {
8897 ""
8898 } else {
8899 ", because:\n"
8900 }
8901 ));
8902 }
8903 if !self.state.deliberation.is_empty() {
8904 text.push_str("\n# What was argued before this vote\n");
8905 for t in self.transcript(&[], j) {
8906 text.push_str(&format!(
8907 "\n## {}{}\n\n{}\n",
8908 t.who,
8909 if t.is_self { " (you)" } else { "" },
8910 t.body.trim()
8911 ));
8912 }
8913 }
8914 text
8915 }
8916
8917 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8924 let mut turns = Vec::new();
8925 for j in &self.state.judgements {
8926 if j.ranking.is_empty() {
8927 continue;
8928 }
8929 let reasons = j
8930 .reasons
8931 .iter()
8932 .map(|(k, v)| format!("- {k}: {v}"))
8933 .collect::<Vec<_>>()
8934 .join("\n");
8935 turns.push(Turn {
8936 who: format!("Judge {} (opening ranking)", j.judge),
8937 is_self: j.judge == self_idx + 1,
8938 body: format!(
8939 "Ranked {}{}{reasons}",
8940 j.ranking.iter().collect::<String>(),
8941 if reasons.is_empty() {
8942 ""
8943 } else {
8944 ", because:\n"
8945 }
8946 ),
8947 });
8948 }
8949 for t in self
8950 .state
8951 .deliberation
8952 .iter()
8953 .flat_map(|r| r.turns.iter())
8954 .chain(current)
8955 {
8956 turns.push(Turn {
8957 who: format!("Judge {}", t.judge),
8958 is_self: t.judge == self_idx + 1,
8959 body: t.body.clone(),
8960 });
8961 }
8962 turns
8963 }
8964}
8965
8966fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8968 agent::has_session(spec.kind, seat, sessions)
8969}
8970
8971fn next_untried_in_roster<'a>(
8992 roster: &'a [AgentSpec],
8993 start: usize,
8994 tried: &BTreeSet<String>,
8995) -> Option<&'a AgentSpec> {
8996 roster
8997 .get(start + 1..)?
8998 .iter()
8999 .find(|s| !tried.contains(&s.id))
9000}
9001
9002fn pick_successor<'a>(
9023 roster: &'a [AgentSpec],
9024 start: usize,
9025 tried: &BTreeSet<String>,
9026 carried: Option<&BTreeSet<String>>,
9027 others: &BTreeSet<String>,
9028) -> Option<&'a AgentSpec> {
9029 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9030 roster
9031 .get(start + 1..)
9032 .and_then(|tail| {
9033 tail.iter()
9034 .filter(free)
9035 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9036 })
9037 .or_else(|| {
9038 carried
9039 .is_some()
9040 .then(|| roster.iter().find(free))
9041 .flatten()
9042 })
9043 .or_else(|| match carried {
9044 Some(c) => next_for_seat(roster, start, tried, c),
9045 None => next_untried_in_roster(roster, start, tried),
9046 })
9047}
9048
9049fn next_for_seat<'a>(
9059 roster: &'a [AgentSpec],
9060 start: usize,
9061 round_tried: &BTreeSet<String>,
9062 carried_failed: &BTreeSet<String>,
9063) -> Option<&'a AgentSpec> {
9064 roster
9065 .get(start + 1..)?
9066 .iter()
9067 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9068 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9069}
9070
9071fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9077 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9078 return spec;
9079 };
9080 let ok = |id: &str| !h.failed.contains(id);
9081 if let Some(last) = h.last_ok.as_deref()
9082 && ok(last)
9083 && let Some(s) = roster.iter().find(|s| s.id == last)
9084 {
9085 return s.clone();
9086 }
9087 if ok(&spec.id) {
9088 return spec;
9089 }
9090 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9091 roster
9092 .get(from + 1..)
9093 .into_iter()
9094 .flatten()
9095 .chain(roster.iter())
9096 .find(|s| ok(&s.id))
9097 .cloned()
9098 .unwrap_or(spec)
9099}
9100
9101pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9105 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9106}
9107
9108const TIMED_OUT: &str = "timed out";
9111
9112impl FailClass {
9113 fn of(out: &AgentOutcome) -> Option<Self> {
9115 match out {
9116 AgentOutcome::Ok(_) => None,
9117 AgentOutcome::Quota(_) => Some(Self::Quota),
9118 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9119 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9120 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9121 AgentOutcome::ResourceBlocked(_) => None,
9124 }
9125 }
9126
9127 fn stem_word(&self) -> &'static str {
9129 match self {
9130 Self::Quota => "quota",
9131 _ => "handover",
9132 }
9133 }
9134}
9135
9136fn failure_signature(msg: &str) -> String {
9140 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9141 if let Some(code) = line
9144 .strip_prefix("process exited with no usable output (exit code ")
9145 .and_then(|s| s.strip_suffix(')'))
9146 .and_then(|s| s.parse::<i32>().ok())
9147 {
9148 line = format!("exited with some({code}) and no usable output");
9149 } else if matches!(
9150 line.as_str(),
9151 "process exited with no usable output (no exit code reported)"
9152 | "process exited with no usable output (exit code unavailable)"
9153 ) {
9154 line = "exited with none and no usable output".to_owned();
9155 }
9156 let mut out = Vec::new();
9157 for word in line.split_whitespace() {
9158 if word.contains('/') || word.contains('\\') {
9159 out.push("<path>".to_owned());
9160 continue;
9161 }
9162 let mut w = String::new();
9163 let mut in_digits = false;
9164 for c in word.chars() {
9165 if c.is_ascii_digit() {
9166 if !in_digits {
9167 w.push('#');
9168 }
9169 in_digits = true;
9170 } else {
9171 in_digits = false;
9172 w.push(c);
9173 }
9174 }
9175 out.push(w);
9176 }
9177 out.join(" ").chars().take(120).collect()
9178}
9179
9180fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9187 match cur {
9188 FailClass::Quota | FailClass::Timeout => true,
9189 FailClass::Other(_) => prev != Some(cur),
9190 }
9191}
9192
9193fn fail_reason(out: &AgentOutcome) -> String {
9195 match out {
9196 AgentOutcome::Ok(_) => String::new(),
9197 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9198 AgentOutcome::Dropped(o) => format!(
9199 "the CLI dropped the stream ({})",
9200 o.dropped
9201 .as_ref()
9202 .map(|d| d.why.as_str())
9203 .unwrap_or("it ended without delivering its answer")
9204 ),
9205 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9206 e.lines().next().unwrap_or("").chars().take(160).collect()
9207 }
9208 }
9209}
9210
9211pub(crate) fn record_handover(
9214 state: &mut RunState,
9215 node: &str,
9216 seat: &str,
9217 from: &str,
9218 to: &str,
9219 class: &FailClass,
9220 reason: &str,
9221) {
9222 let message = if *class == FailClass::Quota {
9223 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9224 } else {
9225 format!("{seat}: handed over {from} -> {to} ({reason})")
9226 };
9227 state.event(node, message);
9228 state.handovers.push(Handover {
9229 at: Timestamp::now(),
9230 node: node.to_owned(),
9231 seat: seat.to_owned(),
9232 from: from.to_owned(),
9233 to: to.to_owned(),
9234 reason: reason.to_owned(),
9235 });
9236}
9237
9238fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9252 commands.iter().any(|c| c.exit_code.is_none())
9253}
9254
9255fn verified_noop_claim(
9268 usable: bool,
9269 commands: &[agent::CommandEvidence],
9270 text: &str,
9271) -> Option<String> {
9272 (usable && !has_unconfirmed_command(commands))
9273 .then(|| verdict::verified_noop(text))
9274 .flatten()
9275}
9276
9277fn short(commit: &str) -> String {
9278 commit.chars().take(7).collect()
9279}
9280
9281fn make_executable(path: &Path) -> Result<()> {
9282 #[cfg(unix)]
9283 {
9284 use std::os::unix::fs::PermissionsExt as _;
9285 let mut perms = std::fs::metadata(path)?.permissions();
9286 perms.set_mode(0o755);
9287 std::fs::set_permissions(path, perms)?;
9288 }
9289 #[cfg(not(unix))]
9290 {
9291 let _ = path;
9292 }
9293 Ok(())
9294}
9295
9296struct WaveCtx<'a> {
9303 run: &'a str,
9306 node: &'a str,
9308 prompts: &'a Prompts,
9309 cache: Option<&'a Path>,
9311 round: Option<usize>,
9314 carry_seats: bool,
9318 pauses: Option<(&'a Pause, &'a Pause)>,
9321}
9322
9323fn repository_wait(
9325 state: &RunState,
9326 pauses: Option<(&Pause, &Pause)>,
9327) -> crate::repo_lock::WaitPolicy {
9328 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9329 let inherited = crate::repo_lock::WaitPolicy::current();
9330 crate::repo_lock::WaitPolicy::new(
9331 Duration::from_secs(state.config.graph.timeout_resource_wait),
9332 move || {
9333 inherited.cancelled()
9334 || pauses
9335 .as_ref()
9336 .is_some_and(|(a, b)| a.parked() || b.parked())
9337 },
9338 )
9339}
9340
9341fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9342 if error.is::<crate::repo_lock::ResourceBlocked>() {
9343 state.resource_block = Some(crate::run::ResourceBlock {
9344 cause: crate::run::ResourceBlockCause::Other,
9345 node: "repository".into(),
9346 kind: if error.to_string().contains("disk pressure") {
9347 "disk pressure"
9348 } else {
9349 "repository coordination"
9350 }
9351 .into(),
9352 detail: error.to_string(),
9353 at: Timestamp::now(),
9354 });
9355 state.event("resource", error.to_string());
9356 }
9357}
9358
9359async fn coordinate_operation(
9360 state: &mut RunState,
9361 cwd: &Path,
9362 wait: &crate::repo_lock::WaitPolicy,
9363) -> Result<Option<crate::repo_lock::SeatLease>> {
9364 let result =
9365 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9366 .await;
9367 if let Err(e) = &result {
9368 record_repository_block(state, e);
9369 }
9370 result
9371}
9372
9373async fn run_one(
9375 job: SeatJob,
9376 sem: Arc<Semaphore>,
9377 ctx: &WaveCtx<'_>,
9378 state: &mut RunState,
9379 attempt: usize,
9380) -> (SeatState, AgentOutcome) {
9381 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9382 .await
9383 .pop()
9384 .expect("one job in, one result out");
9385 (seat, out)
9386}
9387
9388async fn wave(
9394 jobs: Vec<SeatJob>,
9395 sem: Arc<Semaphore>,
9396 ctx: &WaveCtx<'_>,
9397 state: &mut RunState,
9398 attempt: usize,
9399) -> Vec<(usize, SeatState, AgentOutcome)> {
9400 let WaveCtx {
9401 run,
9402 node,
9403 prompts,
9404 cache,
9405 round,
9406 carry_seats: _,
9407 pauses,
9408 } = *ctx;
9409 for job in &jobs {
9410 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9411 }
9412 if let Err(e) = state.save() {
9413 tracing::warn!("could not persist in-progress seats: {e:#}");
9418 }
9419 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9435 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9439 let cache_guard = if let Some(cache_dir) = cache {
9440 if jobs_had_a_writer {
9441 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9442 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9443 match acquire_cache_lease(
9444 state,
9445 cache_dir,
9446 &owner,
9447 resource_limit,
9448 node,
9449 node,
9450 &cancel,
9451 )
9452 .await
9453 {
9454 Ok(Some(g)) => Some(g),
9455 Ok(None) => {
9458 if let Some(block) = &mut state.resource_block {
9459 block.cause = crate::run::ResourceBlockCause::Cancelled {
9460 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9461 };
9462 }
9463 return block_wave(
9464 jobs,
9465 state,
9466 node,
9467 attempt,
9468 "parked while waiting for the shared build cache",
9469 );
9470 }
9471 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9474 Err(e) => {
9475 let why = format!("{e:#}");
9479 state.event(
9480 "resource",
9481 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9482 );
9483 return block_wave(jobs, state, node, attempt, &why);
9484 }
9485 }
9486 } else {
9487 None
9488 }
9489 } else {
9490 None
9491 };
9492 state.seats_spawning(node, attempt);
9495 if let Err(e) = state.save() {
9496 tracing::warn!("could not persist the start of a wave: {e:#}");
9497 }
9498 let slot_limit = resource_limit
9502 + jobs
9503 .iter()
9504 .map(|j| j.timeout)
9505 .max()
9506 .unwrap_or_default()
9507 .saturating_mul(jobs.len() as u32);
9508 let mut set = tokio::task::JoinSet::new();
9509 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9512 let overlay = prompts.overlay(node);
9513 let repository_wait = repository_wait(state, pauses);
9514 for (i, mut job) in jobs.into_iter().enumerate() {
9515 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9516 if cache.is_some() {
9517 job.prompt.push('\n');
9518 job.prompt
9519 .push_str(&prompt::build_cache_note(node, job.allow_write));
9520 }
9521 let sem = Arc::clone(&sem);
9522 let run = run.to_owned();
9523 let node = node.to_owned();
9524 let attachments = if node == "implement" {
9527 state.attachments.clone()
9528 } else {
9529 Vec::new()
9530 };
9531 let cache = cache
9542 .filter(|_| job.allow_write && cache_guard.is_some())
9543 .map(Path::to_path_buf);
9544 let spawned_tx = spawned_tx.clone();
9545 let repository_wait = repository_wait.clone();
9546 let coordination = state.config.coordination.clone();
9547 let fetched_base = state.fetched_base.clone();
9548 set.spawn(async move {
9549 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9553 Ok(p) => p,
9554 Err(_) => {
9555 return (
9556 i,
9557 job.seat,
9558 AgentOutcome::ResourceBlocked(format!(
9559 "no parallel slot became free within {}s",
9560 slot_limit.as_secs()
9561 )),
9562 false,
9563 );
9564 }
9565 };
9566 spawned_tx.send(job.seat.key.clone()).ok();
9567 let mut seat = job.seat;
9568 let out = repository_wait
9569 .scope(agent::invoke(
9570 &job.spec,
9571 &mut seat,
9572 &Invocation {
9573 scope: crate::agent::InvocationScope::Repository,
9574 coordination: coordination.as_ref(),
9575 fetched_base: fetched_base.as_ref(),
9576 cwd: &job.cwd,
9577 prompt: &job.prompt,
9578 timeout: job.timeout,
9579 allow_write: job.allow_write,
9580 unsandboxed: false,
9581 sessions: job.sessions,
9582 artifacts: &job.artifacts,
9583 stem: &job.stem,
9584 run: &run,
9585 node: &node,
9586 cache_dir: cache.as_deref(),
9587 attachments: &attachments,
9588 writable: &[],
9589 },
9590 ))
9591 .await;
9592 let interrupted = out
9593 .as_ref()
9594 .err()
9595 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9596 let out = match out {
9597 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9598 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9599 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9607 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9608 Ok(o) => AgentOutcome::Failed(format!(
9609 "process exited with no usable output ({})",
9610 crate::notices::exit_status(o.exit_code)
9611 )),
9612 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9613 AgentOutcome::ResourceBlocked(e.to_string())
9614 }
9615 Err(e) => AgentOutcome::Failed(e.to_string()),
9616 };
9617 (i, seat, out, interrupted)
9618 });
9619 }
9620 drop(spawned_tx);
9621 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9622 loop {
9623 let joined = tokio::select! {
9624 Some(key) = spawned_rx.recv() => {
9625 state.seat_spawned(&key);
9627 if let Err(e) = state.save() {
9628 tracing::warn!("could not persist a seat's start: {e:#}");
9629 }
9630 continue;
9631 }
9632 joined = set.join_next() => joined,
9633 };
9634 let Some(joined) = joined else {
9635 break;
9636 };
9637 let (i, seat, out, interrupted) = match joined {
9638 Ok(v) => v,
9639 Err(e) => {
9643 tracing::error!("agent task panicked: {e}");
9644 continue;
9645 }
9646 };
9647 state.seat_finished(&seat.key);
9648 if interrupted
9649 && node == "implement"
9650 && let Some(label) = state
9651 .candidates
9652 .iter()
9653 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9654 .map(|candidate| candidate.label)
9655 {
9656 state.candidate_preservation.insert(
9657 label,
9658 "interrupted implementation awaiting persistence".into(),
9659 );
9660 state.interrupted_implementations.insert(label);
9661 }
9662 if let AgentOutcome::ResourceBlocked(why) = &out
9663 && state.resource_block.is_none()
9664 {
9665 state.resource_block = Some(crate::run::ResourceBlock {
9666 cause: crate::run::ResourceBlockCause::Other,
9667 node: node.to_owned(),
9668 kind: if why.contains("disk pressure") {
9669 "disk pressure"
9670 } else if why.contains("repository coordination") {
9671 "repository coordination"
9672 } else {
9673 "parallel slot"
9674 }
9675 .to_owned(),
9676 detail: why.clone(),
9677 at: Timestamp::now(),
9678 });
9679 let activity = if interrupted {
9680 "agent execution interrupted"
9681 } else {
9682 "no agent was started"
9683 };
9684 state.event("resource", format!("{node}: {activity} - {why}"));
9685 }
9686 record_jobs(state, node, round, &seat.key, &out);
9687 if let Err(e) = state.save() {
9688 tracing::warn!("could not persist a seat's completion: {e:#}");
9689 }
9690 if collected.len() <= i {
9691 collected.resize_with(i + 1, || None);
9692 }
9693 collected[i] = Some((i, seat, out));
9694 }
9695 if state
9701 .active
9702 .values()
9703 .any(|a| a.node == node && a.attempt == attempt)
9704 {
9705 state
9706 .active
9707 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9708 if let Err(e) = state.save() {
9709 tracing::warn!("could not persist the end of a wave: {e:#}");
9710 }
9711 }
9712 if let Some(cache_dir) = cache
9719 && jobs_had_a_writer
9720 {
9721 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9722 }
9723 if let Some(guard) = cache_guard {
9724 guard.release();
9725 }
9726 collected.into_iter().flatten().collect()
9727}
9728
9729fn record_jobs(
9740 state: &mut RunState,
9741 node: &str,
9742 round: Option<usize>,
9743 seat: &str,
9744 out: &AgentOutcome,
9745) {
9746 let commands: &[agent::CommandEvidence] = match out {
9747 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9748 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9749 };
9750 let checked_at = Timestamp::now();
9751 for c in commands {
9752 state.jobs.push(JobRecord {
9753 node: node.to_owned(),
9754 round,
9755 seat: seat.to_owned(),
9756 id: c.id.clone(),
9757 description: c.description.clone(),
9758 checked_at,
9759 status: match c.exit_code {
9760 Some(0) => JobStatus::Completed,
9761 Some(_) => JobStatus::Failed,
9762 None => JobStatus::Unknown,
9763 },
9764 exit_code: c.exit_code,
9765 result_summary: c.result_summary.clone(),
9766 source: c.source.clone(),
9767 });
9768 }
9769}
9770
9771fn round_is_clean(
9792 blocking: usize,
9793 e2e_ok: bool,
9794 answered: usize,
9795 expected: usize,
9796 quota_missing: usize,
9797 policy: IncompleteReviewPolicy,
9798) -> bool {
9799 if blocking != 0 || !e2e_ok {
9800 return false;
9801 }
9802 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9803 return true;
9804 }
9805 answered > 0 && expected - answered <= quota_missing
9806}
9807
9808pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9832 if max_rounds == 0
9833 || reviews.last().is_some_and(|r| {
9834 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9835 })
9836 {
9837 return Some(RunStatus::Gating);
9838 }
9839 let last = reviews.last()?;
9840 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9841 if reviews.len() < max_rounds && !stagnant {
9842 return None;
9843 }
9844 if last.incomplete() && last.blocking == 0 {
9845 return Some(RunStatus::Blocked);
9846 }
9847 if last.e2e_status() == E2eStatus::ResourceBlocked
9848 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9849 {
9850 return None;
9851 }
9852 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9853 RunStatus::Gating
9854 } else {
9855 RunStatus::Blocked
9856 })
9857}
9858
9859pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9874 if nudged {
9875 (full / 4).max(Duration::from_secs(120)).min(full)
9876 } else {
9877 full
9878 }
9879}
9880
9881#[allow(clippy::too_many_arguments)]
9910async fn ask_json_wave<T>(
9911 jobs: Vec<SeatJob>,
9912 sem: Arc<Semaphore>,
9913 retries: usize,
9914 roster: &[AgentSpec],
9915 ctx: &WaveCtx<'_>,
9916 losses: &mut Vec<QuotaLoss>,
9917 state: &mut RunState,
9918 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9919) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9920where
9921 T: serde::de::DeserializeOwned + Send + 'static,
9922{
9923 ask_wave_with(
9924 jobs,
9925 sem,
9926 retries,
9927 roster,
9928 ctx,
9929 losses,
9930 state,
9931 &|text: &str| {
9932 let v = verdict::extract_json::<T>(text)?;
9933 validate(&v)?;
9934 Ok(v)
9935 },
9936 )
9937 .await
9938}
9939
9940#[allow(clippy::too_many_arguments)]
9949async fn ask_wave_with<T>(
9950 jobs: Vec<SeatJob>,
9951 sem: Arc<Semaphore>,
9952 retries: usize,
9953 roster: &[AgentSpec],
9954 ctx: &WaveCtx<'_>,
9955 losses: &mut Vec<QuotaLoss>,
9956 state: &mut RunState,
9957 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9958) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9959where
9960 T: Send + 'static,
9961{
9962 let n = jobs.len();
9963 let originals: Vec<SeatJob> = jobs;
9964 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9965 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9966 let mut nudges: Vec<usize> = vec![0; n];
9973 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9977 let mut tried: Vec<BTreeSet<String>> = specs
9978 .iter()
9979 .map(|s| BTreeSet::from([s.id.clone()]))
9980 .collect();
9981 let starts: Vec<usize> = specs
9982 .iter()
9983 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9984 .collect();
9985 let carry = ctx.carry_seats && !roster.is_empty();
9986 let carried: Vec<BTreeSet<String>> = originals
9989 .iter()
9990 .map(|j| {
9991 state
9992 .seat_history
9993 .get(&j.seat.key)
9994 .filter(|_| carry)
9995 .map(|h| h.failed.clone())
9996 .unwrap_or_default()
9997 })
9998 .collect();
9999 let mut prev: Vec<Option<FailClass>> = originals
10000 .iter()
10001 .map(|j| {
10002 state
10003 .seat_history
10004 .get(&j.seat.key)
10005 .filter(|_| carry)
10006 .and_then(|h| h.last_fail.clone())
10007 })
10008 .collect();
10009 let next_agent =
10010 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10011 let others: BTreeSet<String> = specs
10012 .iter()
10013 .enumerate()
10014 .filter(|(j, _)| *j != i)
10015 .map(|(_, s)| s.id.clone())
10016 .collect();
10017 pick_successor(
10018 roster,
10019 starts[i],
10020 tried,
10021 carry.then(|| &carried[i]),
10022 &others,
10023 )
10024 .cloned()
10025 };
10026 let mut fresh: Vec<Option<String>> = vec![None; n];
10027 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10028 let mut pending: Vec<usize> = (0..n).collect();
10029
10030 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10035 for round in 0..max_rounds {
10036 if pending.is_empty() {
10037 break;
10038 }
10039 let mut batch = Vec::with_capacity(pending.len());
10040 let mut renudged: Vec<&str> = Vec::new();
10041 for &i in &pending {
10042 let src = &originals[i];
10043 let full: &str = match &src.handover {
10048 Some(h) if specs[i].id != src.spec.id => h,
10049 _ => &src.prompt,
10050 };
10051 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10054 (
10055 full.to_owned(),
10056 src.timeout,
10057 format!("{}-{word}-{}", src.stem, specs[i].id),
10058 )
10059 } else if nudges[i] == 0 {
10060 (full.to_owned(), src.timeout, src.stem.clone())
10061 } else {
10062 renudged.push(src.seat.key.as_str());
10063 let why = done[i]
10064 .as_ref()
10065 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10066 .unwrap_or_else(|| "no parsable answer".to_owned());
10067 let nudge = prompt::nudge(&why);
10068 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10069 let prompt = if nudged {
10070 nudge
10071 } else {
10072 format!("{full}\n\n---\n\n{nudge}")
10073 };
10074 (
10075 prompt,
10076 retry_budget(src.timeout, nudged),
10077 format!("{}-retry{}", src.stem, nudges[i]),
10078 )
10079 };
10080 batch.push(SeatJob {
10081 spec: specs[i].clone(),
10082 seat: seats[i].clone(),
10083 cwd: src.cwd.clone(),
10084 prompt,
10085 timeout,
10086 allow_write: src.allow_write,
10087 sessions: src.sessions,
10088 artifacts: src.artifacts.clone(),
10089 stem,
10090 handover: None,
10091 });
10092 }
10093
10094 if !renudged.is_empty() {
10095 state.event(
10096 ctx.node,
10097 format!("retry {round}: re-asking {}", renudged.join(", ")),
10098 );
10099 }
10100 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10101 let mut still = Vec::new();
10102 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10103 seats[i] = seat;
10104 if let AgentOutcome::ResourceBlocked(why) = &out {
10107 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10108 continue;
10109 }
10110 let class = FailClass::of(&out);
10111 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10114 && nudges[i] < retries
10115 && !roster.is_empty();
10116 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10117 let next = should_hand_over(prev[i].as_ref(), &cur)
10118 .then(|| next_agent(i, &tried[i], &specs))
10119 .flatten();
10120 if carry {
10121 let h = state
10122 .seat_history
10123 .entry(originals[i].seat.key.clone())
10124 .or_default();
10125 h.failed.insert(specs[i].id.clone());
10126 h.last_fail = Some(cur.clone());
10127 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10128 h.last_ok = None;
10129 }
10130 if let Err(e) = state.save() {
10133 tracing::warn!("could not persist a seat's failure history: {e:#}");
10134 }
10135 }
10136 if let Some(next) = next {
10137 record_handover(
10138 state,
10139 ctx.node,
10140 &originals[i].seat.key,
10141 &specs[i].id,
10142 &next.id,
10143 &cur,
10144 &fail_reason(&out),
10145 );
10146 tried[i].insert(next.id.clone());
10147 prev[i] = Some(cur.clone());
10148 seats[i] =
10149 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10150 specs[i] = next;
10151 fresh[i] = Some(cur.stem_word().to_owned());
10152 nudges[i] = 0;
10153 done[i] = Some(Err(anyhow::anyhow!(
10154 "handed over after: {}",
10155 fail_reason(&out)
10156 )));
10157 still.push(i);
10158 continue;
10159 }
10160 }
10161 if carry
10162 && !nudge_first
10163 && let Some(cur) = class.clone()
10164 {
10165 let h = state
10167 .seat_history
10168 .entry(originals[i].seat.key.clone())
10169 .or_default();
10170 h.failed.insert(specs[i].id.clone());
10171 h.last_fail = Some(cur);
10172 }
10173 let parsed = match out {
10174 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10175 AgentOutcome::Quota(o) => {
10176 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10177 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10178 }
10179 AgentOutcome::Dropped(o) => {
10183 let why = o
10184 .dropped
10185 .as_ref()
10186 .map(|d| d.why.as_str())
10187 .unwrap_or("the CLI ended the stream without delivering its answer");
10188 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10189 }
10190 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10191 Err(anyhow::anyhow!(e))
10192 }
10193 };
10194 let quota = class == Some(FailClass::Quota);
10195 let failed = parsed.is_err();
10196 done[i] = Some(parsed);
10197 if carry && !failed {
10198 let h = state
10199 .seat_history
10200 .entry(originals[i].seat.key.clone())
10201 .or_default();
10202 h.failed.remove(&specs[i].id);
10203 h.last_ok = Some(specs[i].id.clone());
10204 h.last_fail = None;
10205 }
10206 let agent_failure = class.is_some()
10214 && !nudge_first
10215 && !roster.is_empty()
10216 && next_agent(i, &tried[i], &specs).is_some();
10217 if failed && !quota && !agent_failure && nudges[i] < retries {
10218 nudges[i] += 1;
10219 still.push(i);
10220 }
10221 }
10222 pending = still;
10223 }
10224
10225 for (i, q) in last_quota.into_iter().enumerate() {
10228 if let Some(reset) = q {
10229 losses.push(QuotaLoss {
10230 seat: originals[i].seat.key.clone(),
10231 node: ctx.node.to_owned(),
10232 at: Timestamp::now(),
10233 reset,
10234 });
10235 }
10236 }
10237
10238 seats
10239 .into_iter()
10240 .zip(done)
10241 .zip(nudges)
10242 .map(|((seat, res), attempts)| {
10243 (
10244 seat,
10245 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10246 attempts,
10247 )
10248 })
10249 .collect()
10250}
10251
10252fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10254 results
10255 .iter()
10256 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10257}
10258
10259fn block_wave(
10263 jobs: Vec<SeatJob>,
10264 state: &mut RunState,
10265 node: &str,
10266 attempt: usize,
10267 why: &str,
10268) -> Vec<(usize, SeatState, AgentOutcome)> {
10269 state.resource_wait = None;
10270 state
10271 .active
10272 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10273 if let Err(e) = state.save() {
10274 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10275 }
10276 jobs.into_iter()
10277 .enumerate()
10278 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10279 .collect()
10280}
10281
10282async fn acquire_cache_lease(
10295 state: &mut RunState,
10296 cache_dir: &Path,
10297 owner: &crate::cache::Owner,
10298 budget: Duration,
10299 node: &str,
10300 context: &str,
10301 cancel: &(dyn Fn() -> bool + Sync),
10302) -> Result<Option<crate::cache::Guard>> {
10303 let home = crate::run::home();
10304 let started = Instant::now();
10305 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10306 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10307 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10308 Err(e) => {
10309 state.event(
10310 "verify",
10311 format!("{context}: could not check the shared build cache: {e:#}"),
10312 );
10313 if let Err(e2) = state.save() {
10314 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10315 }
10316 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10317 }
10318 };
10319 state.event(
10320 "verify",
10321 format!(
10322 "{context}: waiting for the shared build cache at {} ({})",
10323 cache_dir.display(),
10324 busy.describe()
10325 ),
10326 );
10327 state.resource_wait = Some(crate::run::ResourceWait {
10330 node: context.to_owned(),
10331 kind: "shared build cache".to_owned(),
10332 holder: busy.describe(),
10333 since: Timestamp::now(),
10334 limit_secs: budget.as_secs(),
10335 });
10336 if let Err(e) = state.save() {
10337 tracing::warn!("could not persist a cache wait: {e:#}");
10338 }
10339 let remaining = budget.saturating_sub(started.elapsed());
10340 let waited = crate::cache::wait_for_unless(
10341 &home,
10342 cache_dir,
10343 owner,
10344 remaining,
10345 Duration::from_secs(5),
10346 cancel,
10347 )
10348 .await;
10349 state.resource_wait = None;
10350 let cause = match &waited {
10351 Ok(Some(_)) => None,
10352 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10353 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10354 Some(crate::run::ResourceBlockCause::Timeout {
10355 waited_secs: started.elapsed().as_secs(),
10356 limit_secs: budget.as_secs(),
10357 })
10358 }
10359 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10360 };
10361 if let Some(cause) = cause {
10362 let detail = match &waited {
10363 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10364 Err(error) => format!("{error:#}"),
10365 };
10366 state.resource_block = Some(crate::run::ResourceBlock {
10367 cause,
10368 node: node.to_owned(),
10369 kind: "shared build cache".into(),
10370 detail,
10371 at: Timestamp::now(),
10372 });
10373 state.event(
10374 "resource",
10375 state.resource_block.as_ref().unwrap().description(),
10376 );
10377 }
10378 if let Err(error) = state.save() {
10379 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10380 }
10381 waited
10382}
10383
10384#[derive(Debug)]
10389struct CacheCheckError(String);
10390
10391impl std::fmt::Display for CacheCheckError {
10392 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10393 f.write_str(&self.0)
10394 }
10395}
10396
10397impl std::error::Error for CacheCheckError {}
10398
10399fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10401 state
10402 .resource_block
10403 .get_or_insert_with(|| crate::run::ResourceBlock {
10404 cause: crate::run::ResourceBlockCause::Other,
10405 node: node.to_owned(),
10406 kind: "shared build cache".into(),
10407 detail: format!("{error:#}"),
10408 at: Timestamp::now(),
10409 });
10410 if let Err(error) = state.save() {
10411 tracing::warn!("could not persist cache resource block: {error:#}");
10412 }
10413}
10414
10415#[allow(clippy::too_many_arguments)]
10436async fn with_cache_lease<'s, F, Fut>(
10437 state: &'s mut RunState,
10438 cache_dir: Option<&Path>,
10439 node: &str,
10440 seat: &str,
10441 worktree: &Path,
10442 head: &str,
10443 budget: Duration,
10444 context: &str,
10445 wait: &crate::repo_lock::WaitPolicy,
10446 body: F,
10447) -> Result<(Vec<CommandOutcome>, bool)>
10448where
10449 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10450 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10451{
10452 let started = Instant::now();
10455 let home = crate::run::home();
10456 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10457 let guard = if let Some(cache_dir) = cache_dir {
10458 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10459 match acquire_cache_lease(
10460 state,
10461 cache_dir,
10462 &owner,
10463 resource_limit,
10464 node,
10465 context,
10466 &|| wait.cancelled(),
10467 )
10468 .await
10469 .and_then(|g| {
10470 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10471 }) {
10472 Ok(g) => Some(g),
10473 Err(e) => {
10474 record_cache_error(state, node, &e);
10475 return Ok((
10476 vec![CommandOutcome {
10477 command: "(waiting for the shared build cache)".into(),
10478 code: None,
10479 output_tail: e.to_string(),
10480 duration_ms: started.elapsed().as_millis() as u64,
10481 resource_blocked: true,
10482 }],
10483 false,
10484 ));
10485 }
10486 }
10487 } else {
10488 None
10489 };
10490 let _coordination = coordinate_operation(state, worktree, wait).await?;
10491 if let Some(cache_dir) = cache_dir {
10492 let identity = crate::cache::Identity::new(worktree, head);
10493 if let Err(e) = crate::cache::ensure_fresh(
10494 &home,
10495 cache_dir,
10496 &identity,
10497 &state.config.verify.cargo_manifests(),
10498 ) {
10499 record_cache_error(state, node, &e);
10500 state.event(
10501 "verify",
10502 format!(
10503 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10504 worktree.display(),
10505 short(head)
10506 ),
10507 );
10508 if let Some(guard) = guard {
10509 guard.release();
10510 }
10511 return Ok((
10512 vec![CommandOutcome {
10513 command: "(confirming the shared build cache is fresh)".into(),
10514 code: None,
10515 output_tail: e.to_string(),
10516 duration_ms: started.elapsed().as_millis() as u64,
10517 resource_blocked: true,
10518 }],
10519 false,
10520 ));
10521 }
10522 }
10523 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10524 if !timed_out_pids.is_empty() {
10525 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10526 }
10527 drop(_coordination);
10530 if let Some(guard) = guard {
10531 guard.release();
10532 }
10533 Ok((outcomes, retried))
10534}
10535
10536async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10548 wait_for_pids_with(
10549 pids,
10550 crate::proc::pid_alive,
10551 LEASE_RELEASE_POLL,
10552 LEASE_RELEASE_MAX_WAIT,
10553 )
10554 .await;
10555}
10556
10557async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10563 pids: &[u32],
10564 alive: F,
10565 poll: Duration,
10566 max_wait: Duration,
10567) {
10568 let deadline = Instant::now() + max_wait;
10569 loop {
10570 if pids.iter().all(|&pid| !alive(pid)) {
10571 return;
10572 }
10573 if Instant::now() >= deadline {
10574 return;
10575 }
10576 tokio::time::sleep(poll).await;
10577 }
10578}
10579
10580fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10587 outcomes.iter().any(|o| o.resource_blocked)
10588}
10589
10590pub(crate) enum GateFix {
10592 Retry,
10594 Stop,
10597 Defer,
10600}
10601
10602fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10610 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10611 red.peek().is_some()
10612 && red.all(|o| {
10613 !o.resource_blocked
10614 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10615 && !o.output_tail.trim().is_empty()
10616 })
10617}
10618
10619fn e2e_outcome_label(o: &CommandOutcome) -> String {
10623 if o.ok() {
10624 return "pass".to_owned();
10625 }
10626 let reason = if o.build_failed() {
10627 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10628 } else {
10629 format!("FAIL ({:?})", o.code)
10630 };
10631 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10632}
10633
10634async fn run_e2e_with_retry(
10642 state: &mut RunState,
10643 shell: &[String],
10644 commands: &[String],
10645 worktree: &Path,
10646 timeout: Duration,
10647 context: &str,
10648) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10649 let (mut e2e, mut timed_out_pids) = run_commands(
10650 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10651 )
10652 .await;
10653 for o in &e2e {
10654 state.event(
10655 "verify",
10656 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10657 );
10658 }
10659 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10663 if verify_retried {
10664 state.event(
10665 "verify",
10666 format!(
10667 "{context}: verify could not build/link, not a test result — retrying once \
10668 before concluding"
10669 ),
10670 );
10671 let retried = run_commands(
10672 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10673 )
10674 .await;
10675 e2e = retried.0;
10676 timed_out_pids.extend(retried.1);
10679 for o in &e2e {
10680 state.event(
10681 "verify",
10682 format!(
10683 "{context}: retry `{}` -> {}",
10684 o.command,
10685 e2e_outcome_label(o)
10686 ),
10687 );
10688 }
10689 }
10690 (e2e, verify_retried, timed_out_pids)
10691}
10692
10693#[allow(clippy::too_many_arguments)]
10709async fn run_commands(
10710 state: &mut RunState,
10711 node: &str,
10712 task: &str,
10713 attempt: usize,
10714 shell: &[String],
10715 commands: &[String],
10716 cwd: &Path,
10717 timeout: Duration,
10718) -> (Vec<CommandOutcome>, Vec<u32>) {
10719 if commands.is_empty() {
10720 return (Vec::new(), Vec::new());
10725 }
10726 let mut out = Vec::new();
10727 let mut timed_out_pids = Vec::new();
10728 let total = commands.len();
10729 for (idx, command) in commands.iter().enumerate() {
10730 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10731 if let Err(e) = state.save() {
10732 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10733 }
10734 let started = Instant::now();
10735 let mut cmd = tokio::process::Command::new(&shell[0]);
10736 cmd.quiet();
10737 cmd.args(&shell[1..])
10738 .arg(command)
10739 .current_dir(cwd)
10740 .stdin(std::process::Stdio::null())
10741 .stdout(std::process::Stdio::piped())
10742 .stderr(std::process::Stdio::piped())
10743 .kill_on_drop(true);
10744 let mut outputs = Vec::new();
10745 let protected = (|| -> Result<()> {
10746 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10747 outputs.push(guard);
10748 }
10749 if let Some(cache) = state.config.cache_dir() {
10750 outputs.push(crate::retention::OutputLease::cache(
10751 &crate::run::home(),
10752 &cache,
10753 false,
10754 )?);
10755 }
10756 Ok(())
10757 })();
10758 if let Err(e) = protected {
10759 out.push(CommandOutcome {
10760 command: command.clone(),
10761 code: None,
10762 output_tail: format!("output ownership unavailable: {e:#}"),
10763 duration_ms: 0,
10764 resource_blocked: true,
10765 });
10766 continue;
10767 }
10768 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10769 let (code, body) = match spawned {
10770 Ok(mut running) => {
10771 let pid = running.child().id();
10772 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10773 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10774 let mut paths = vec![cwd.to_owned()];
10775 if let Some(cache) = state.config.cache_dir() {
10776 paths.push(cache);
10777 }
10778 let mut pressure = None;
10779 let status = tokio::select! {
10780 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10781 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10782 pressure = Some(reason); None
10783 }
10784 };
10785 let (code, failure) = match status {
10786 Some(Ok(Ok(status))) => (
10787 status.code(),
10788 status
10789 .code()
10790 .is_none()
10791 .then(|| format!("command terminated: {status}")),
10792 ),
10793 Some(Ok(Err(error))) => {
10794 (None, Some(format!("failed waiting for command: {error}")))
10795 }
10796 Some(Err(_)) => {
10797 if !running.coordinated()
10801 && let Some(pid) = pid
10802 {
10803 timed_out_pids.push(pid);
10804 }
10805 (
10806 None,
10807 Some(format!("timed out after {}s", timeout.as_secs())),
10808 )
10809 }
10810 None => (None, None),
10811 };
10812 running.finish().await;
10813 if let Some(reason) = pressure {
10814 state.resource_block = Some(crate::run::ResourceBlock {
10815 cause: crate::run::ResourceBlockCause::Other,
10816 node: node.to_owned(),
10817 kind: "disk pressure".to_owned(),
10818 detail: reason.clone(),
10819 at: Timestamp::now(),
10820 });
10821 out.push(CommandOutcome {
10822 command: command.clone(),
10823 code: None,
10824 output_tail: reason,
10825 duration_ms: started.elapsed().as_millis() as u64,
10826 resource_blocked: true,
10827 });
10828 break;
10829 }
10830 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10831 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10832 if let Some(failure) = failure {
10833 body.push('\n');
10834 body.push_str(&failure);
10835 }
10836 (code, body)
10837 }
10838 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10839 };
10840 out.push(CommandOutcome {
10841 command: command.clone(),
10842 code,
10843 output_tail: tail(&body, OUTPUT_TAIL),
10844 duration_ms: started.elapsed().as_millis() as u64,
10845 resource_blocked: false,
10846 });
10847 }
10848 state.task_finished(task);
10849 if let Err(e) = state.save() {
10850 tracing::warn!("could not persist the end of {task}: {e:#}");
10851 }
10852 (out, timed_out_pids)
10853}
10854
10855fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10867 let repo = repo.display();
10868 match style {
10869 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10870 MergeStyle::Squash => {
10871 let subject = message
10874 .lines()
10875 .next()
10876 .unwrap_or(branch)
10877 .replace(['\\', '"', '$', '`'], "");
10878 format!(
10879 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10880 )
10881 }
10882 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10883 }
10884}
10885
10886const PR_TITLE_MAX: usize = 240;
10897
10898fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10909 let line = queue::first_line(text)?;
10910 let chars: Vec<char> = line.chars().collect();
10911 let (kept, mark) = if chars.len() <= max {
10912 (line.to_owned(), "")
10913 } else if let Some(end) = sentence_end(&chars, max) {
10914 (chars[..end].iter().collect::<String>(), "")
10915 } else {
10916 let room = max.saturating_sub(3);
10917 let cut = if chars[room].is_whitespace() {
10920 room
10921 } else {
10922 chars[..room]
10923 .iter()
10924 .rposition(|c| c.is_whitespace())
10925 .unwrap_or(room)
10926 };
10927 let head: String = chars[..cut].iter().collect();
10928 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10929 (head.to_owned(), "...")
10930 };
10931 if kept.is_empty() {
10932 return None;
10933 }
10934 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10935 return None;
10936 }
10937 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10938 return None;
10939 }
10940 Some(format!("{kept}{mark}"))
10941}
10942
10943fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10947 const MIN: usize = 20;
10948 for i in MIN..max.min(chars.len()) {
10949 if !matches!(chars[i], '.' | '!' | '?') {
10950 continue;
10951 }
10952 let Some(&next) = chars.get(i + 1) else {
10953 continue;
10954 };
10955 if !next.is_whitespace() {
10956 continue;
10957 }
10958 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10959 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10960 continue;
10961 }
10962 let word: String = chars[..i]
10963 .iter()
10964 .rev()
10965 .take_while(|c| !c.is_whitespace())
10966 .collect::<Vec<_>>()
10967 .into_iter()
10968 .rev()
10969 .collect();
10970 let word = word.to_ascii_lowercase();
10971 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10972 continue;
10973 }
10974 let end = chars[..i]
10975 .iter()
10976 .rposition(|c| !c.is_whitespace())
10977 .map_or(i, |p| p + 1);
10978 return Some(end);
10979 }
10980 None
10981}
10982
10983struct PrMessage {
10988 title: String,
10989 body: String,
10990}
10991
10992impl PrMessage {
10993 fn commit_message(&self) -> String {
10997 format!("{}\n\n{}", self.title, self.body)
10998 }
10999}
11000
11001fn title_marker(line: &str) -> Option<&str> {
11003 let line = line.trim();
11004 let head = line.get(..6)?;
11005 head.eq_ignore_ascii_case("title:")
11006 .then(|| line[6..].trim())
11007}
11008
11009fn summary_title(summary: &str) -> Option<String> {
11014 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11015 let raw = title_marker(first)?;
11016 if raw.is_empty() {
11017 return None;
11018 }
11019 let title = queue::title_from(raw, PR_TITLE_MAX);
11020 let lower = title.to_ascii_lowercase();
11021 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11022 return None;
11023 }
11024 Some(title)
11025}
11026
11027const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11029
11030pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11032
11033pub(crate) fn is_review_run(state: &RunState) -> bool {
11037 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11038}
11039
11040fn review_title(state: &RunState) -> Option<String> {
11045 english_subject(
11046 state.reviewed_commits.as_ref()?.first()?,
11047 state.config.graph.github_english(),
11048 )
11049}
11050
11051fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11054 let raw = raw.trim();
11055 if raw.is_empty() {
11056 return None;
11057 }
11058 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11059 return None;
11060 }
11061 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11062 return None;
11063 }
11064 let title = queue::title_from(raw, PR_TITLE_MAX);
11065 let lower = title.to_ascii_lowercase();
11066 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11067 return None;
11068 }
11069 Some(title)
11070}
11071
11072#[derive(Debug, Clone, PartialEq, Eq)]
11075struct BranchFacts {
11076 commits: Vec<(String, String)>,
11078 stat: String,
11080}
11081
11082const STAT_FILE_LINES: usize = 25;
11084const COMMITS_MAX_CHARS: usize = 20_000;
11086
11087async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11090 let commits = git::commit_log(repo, base, branch).await.ok()?;
11091 if commits.is_empty() {
11092 return None;
11093 }
11094 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11095 let lines: Vec<&str> = stat.lines().collect();
11096 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11097 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11098 let more = format!(" ... {omitted} more file(s)");
11099 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11100 kept.push(&more);
11101 kept.push(lines[lines.len() - 1]);
11102 kept.join("\n")
11103 } else {
11104 lines.join("\n")
11105 };
11106 Some(BranchFacts { commits, stat })
11107}
11108
11109fn markdown_safe(text: &str) -> String {
11112 text.replace("</details>", "</details>")
11113 .replace("\x60\x60\x60", "~~~")
11114}
11115
11116fn neutral_title(state: &RunState, winner: char) -> String {
11117 format!(
11118 "chore: land candidate {} of run {}",
11119 winner.to_ascii_uppercase(),
11120 state.id
11121 )
11122}
11123
11124pub(crate) async fn repair_landing_gate(
11127 state: &mut RunState,
11128 winner: &Candidate,
11129 outcomes: &[CommandOutcome],
11130) -> Result<GateFix> {
11131 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11132 return Ok(GateFix::Stop);
11133 }
11134 let mut runner = Runner {
11135 roles: state.config.resolve_roles()?,
11136 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11137 state: state.clone(),
11138 pause: Pause::new(),
11139 interrupt: Pause::new(),
11140 };
11141 let result = runner.gate_fix_round(winner, outcomes).await;
11142 *state = runner.state;
11143 result
11144}
11145
11146pub(crate) async fn check_gate_repair_e2e(
11149 state: &mut RunState,
11150 winner: &Candidate,
11151 wait: &crate::repo_lock::WaitPolicy,
11152) -> Result<GateFix> {
11153 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11154 return Ok(GateFix::Retry);
11155 }
11156 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11157 if state
11158 .gate_repair_e2e
11159 .as_ref()
11160 .is_some_and(|r| r.head == head && r.passed)
11161 {
11162 return Ok(GateFix::Retry);
11163 }
11164 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11165 head: head.clone(),
11166 passed: false,
11167 outcomes: Vec::new(),
11168 });
11169 state.save()?;
11170 let commands = state.config.verify.e2e.clone();
11171 let outcomes = if commands.is_empty() {
11172 Vec::new()
11173 } else {
11174 let shell = state.config.shell();
11175 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11176 let cache_dir = state.config.cache_dir();
11177 let (outcomes, _) = with_cache_lease(
11178 state,
11179 cache_dir.as_deref(),
11180 "e2e",
11181 "e2e",
11182 &winner.worktree,
11183 &head,
11184 timeout,
11185 "gate repair e2e",
11186 wait,
11187 |state, budget| async move {
11188 run_e2e_with_retry(
11189 state,
11190 &shell,
11191 &commands,
11192 &winner.worktree,
11193 budget,
11194 "gate repair e2e",
11195 )
11196 .await
11197 },
11198 )
11199 .await?;
11200 outcomes
11201 };
11202 let deferred = verify_inconclusive(&outcomes);
11203 let passed = outcomes.iter().all(CommandOutcome::ok)
11204 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11205 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11206 head,
11207 passed,
11208 outcomes,
11209 });
11210 state.save()?;
11211 if deferred {
11212 Ok(GateFix::Defer)
11213 } else if !passed {
11214 state.event(
11215 "gate",
11216 "verify.e2e failed after gate repair; nothing published",
11217 );
11218 Ok(GateFix::Stop)
11219 } else {
11220 Ok(GateFix::Retry)
11221 }
11222}
11223
11224pub(crate) async fn run_landing_gate(
11226 state: &mut RunState,
11227 winner: &Candidate,
11228 wait: &crate::repo_lock::WaitPolicy,
11229) -> Result<Vec<CommandOutcome>> {
11230 let shell = state.config.shell();
11231 let gate_commands = state.config.verify.gate.clone();
11232 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11233 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11234 Vec::new()
11235 } else {
11236 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11237 let cache_dir = state.config.cache_dir();
11238 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11239 let (outcomes, _) = with_cache_lease(
11240 state,
11241 cache_dir.as_deref(),
11242 "gate",
11243 "gate",
11244 &winner.worktree,
11245 &head,
11246 timeout,
11247 "final gate",
11248 wait,
11249 |state, budget| {
11250 let shell = shell.clone();
11251 let gate_commands = gate_commands.clone();
11252 let winner = winner.clone();
11253 async move {
11254 Runner::run_pre_gate(state, &winner).await;
11255 let (outcomes, timed_out_pids) = run_commands(
11256 state,
11257 "gate",
11258 "gate",
11259 0,
11260 &shell,
11261 &gate_commands,
11262 &winner.worktree,
11263 budget,
11264 )
11265 .await;
11266 (outcomes, false, timed_out_pids)
11267 }
11268 },
11269 )
11270 .await?;
11271 outcomes
11272 };
11273 if outcomes.is_empty() {
11274 state.event("gate", "no gate commands configured; not tested");
11279 }
11280 for o in &outcomes {
11281 state.event(
11282 "gate",
11283 format!(
11284 "`{}` -> {}",
11285 o.command,
11286 if o.ok() {
11287 "pass".to_owned()
11288 } else {
11289 format!(
11290 "FAIL ({:?})\n{}",
11291 o.code,
11292 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11293 )
11294 }
11295 ),
11296 );
11297 }
11298 Ok(outcomes)
11299}
11300
11301pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11307 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11308 ""
11309 } else {
11310 pr_title
11311 }
11312}
11313
11314pub fn landing_subject_source(state: &RunState) -> String {
11318 if is_review_run(state) {
11319 let winner = state.candidates.first().map_or('A', |c| c.label);
11320 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11321 }
11322 state.instruction.clone()
11323}
11324
11325fn summary_without_title(summary: &str) -> String {
11328 let mut lines = summary.trim().lines().peekable();
11329 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11330 lines.next();
11331 }
11332 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11333}
11334
11335#[cfg(test)]
11347fn pr_message(state: &RunState, winner: char) -> PrMessage {
11348 pr_message_with(state, winner, None)
11349}
11350
11351#[cfg(test)]
11354fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11355 let raw = pr_message_raw(
11356 state,
11357 winner,
11358 facts,
11359 crate::pr_outcome::FinalHead::Unreported,
11360 ChangeNote::NotAsked,
11361 );
11362 let id = crate::scrub::Identity::current();
11363 PrMessage {
11364 title: crate::scrub::scrub(&raw.title, &id),
11365 body: crate::scrub::scrub(&raw.body, &id),
11366 }
11367}
11368
11369#[derive(Debug, Clone, Copy)]
11371enum ChangeNote<'a> {
11372 NotAsked,
11374 Summary(&'a crate::run::ChangeSummary),
11376 Unavailable(&'a str),
11378}
11379
11380fn pr_message_raw(
11381 state: &RunState,
11382 winner: char,
11383 facts: Option<&BranchFacts>,
11384 head: crate::pr_outcome::FinalHead<'_>,
11385 change: ChangeNote<'_>,
11386) -> PrMessage {
11387 if !matches!(change, ChangeNote::NotAsked) {
11388 let (title, mut body) = match change {
11389 ChangeNote::Summary(c) => (
11390 c.title.clone(),
11391 format!("## Summary\n\n{}\n\n", c.body.trim()),
11392 ),
11393 ChangeNote::Unavailable(why) => (
11394 neutral_title(state, winner),
11395 format!(
11396 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11397 ),
11398 ),
11399 ChangeNote::NotAsked => unreachable!(),
11400 };
11401 body.push_str(&public_pr_evidence(state, head));
11402 body.push_str(&format!(
11403 "\n---\nmagi:run/{} magi:candidate-{}\n",
11404 state.id,
11405 winner.to_ascii_lowercase()
11406 ));
11407 return PrMessage { title, body };
11408 }
11409 pr_preview(state, winner, facts, head)
11410}
11411
11412fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11415 let mut body = crate::pr_outcome::review_rounds(state);
11416 body.push_str(&crate::pr_outcome::final_verification(state, head));
11417 let open = state.last_round_findings();
11418 if !open.is_empty() {
11419 body.push_str("## Open review findings\n\n");
11420 for f in open {
11421 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11422 }
11423 body.push('\n');
11424 }
11425 if let Some(c) = state.change_summary.as_ref()
11426 && head == crate::pr_outcome::FinalHead::At(&c.head)
11427 {
11428 body.push_str(&format!(
11429 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11430 c.base, c.head
11431 ));
11432 }
11433 body
11434}
11435
11436fn pr_preview(
11438 state: &RunState,
11439 winner: char,
11440 facts: Option<&BranchFacts>,
11441 head: crate::pr_outcome::FinalHead<'_>,
11442) -> PrMessage {
11443 let summary = state
11444 .candidates
11445 .iter()
11446 .find(|c| c.label == winner)
11447 .map(|c| c.summary.as_str())
11448 .unwrap_or_default();
11449 let review = is_review_run(state);
11452 let title = if review {
11453 facts
11454 .and_then(|f| {
11455 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11456 })
11457 .or_else(|| review_title(state))
11458 .or_else(|| {
11459 state
11460 .candidates
11461 .iter()
11462 .find(|c| c.label == winner)
11463 .filter(|c| !c.branch.starts_with("magi/"))
11464 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11465 })
11466 .unwrap_or_else(|| neutral_title(state, winner))
11467 } else {
11468 summary_title(summary).unwrap_or_else(|| {
11469 english_title(
11470 &state.instruction,
11471 PR_TITLE_MAX,
11472 state.config.graph.github_english(),
11473 )
11474 .unwrap_or_else(|| neutral_title(state, winner))
11475 })
11476 };
11477
11478 let mut body = String::new();
11479 let what = summary_without_title(summary);
11480 if !what.is_empty() {
11481 body.push_str("## Summary\n\n");
11482 body.push_str(&what);
11483 body.push_str("\n\n");
11484 }
11485
11486 body.push_str(&crate::pr_outcome::review_rounds(state));
11488 body.push_str(&crate::pr_outcome::final_verification(state, head));
11489 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11490
11491 let open = state.open_findings();
11492 if !open.is_empty() {
11493 body.push_str("## Open review findings\n\n");
11494 for f in &open {
11495 body.push_str(&format!(
11496 "- `{}` [{:?}] {}\n",
11497 f.id, f.severity, "Context retained in local review records."
11498 ));
11499 }
11500 body.push('\n');
11501 }
11502
11503 if let Some(fix) = fix
11504 && !fix.rejected.is_empty()
11505 {
11506 body.push_str("## Declined by the fixer\n\n");
11507 for r in &fix.rejected {
11508 body.push_str(&format!(
11509 "- `{}`: {}\n",
11510 r.id, "Reason retained in local review records."
11511 ));
11512 }
11513 body.push('\n');
11514 }
11515
11516 if review {
11517 body.push_str("## Commits under review\n\n");
11519 if let Some(facts) = facts {
11520 let mut left = COMMITS_MAX_CHARS;
11521 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11522 let mut entry = format!("- {}\n", markdown_safe(subject));
11523 for l in markdown_safe(text).lines() {
11524 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11525 }
11526 if left == 0 {
11527 body.push_str(&format!(
11528 "- ... {} more commit(s)\n",
11529 facts.commits.len() - i
11530 ));
11531 break;
11532 }
11533 if entry.len() > left {
11534 let mut end = left;
11537 while !entry.is_char_boundary(end) {
11538 end -= 1;
11539 }
11540 entry.truncate(end);
11541 entry.push_str("\n ... (truncated)\n");
11542 left = 0;
11543 } else {
11544 left -= entry.len();
11545 }
11546 body.push_str(&entry);
11547 }
11548 if !facts.stat.trim().is_empty() {
11549 body.push_str(&format!(
11550 "\n## Diff stat\n\n```\n{}\n```\n",
11551 markdown_safe(facts.stat.trim())
11552 ));
11553 }
11554 } else {
11555 match &state.reviewed_commits {
11556 Some(subjects) => {
11557 for s in subjects {
11558 body.push_str(&format!("- {}\n", s.trim()));
11559 }
11560 }
11561 None => {
11562 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11565 body.push_str(rest.trim());
11566 body.push('\n');
11567 }
11568 }
11569 }
11570 } else {
11571 let task = state.instruction.trim();
11572 let task = if task.is_empty() {
11573 "(empty task)"
11574 } else {
11575 task
11576 };
11577 body.push_str(&format!(
11578 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11579 task.replace("</details>", "</details>")
11580 ));
11581 }
11582
11583 body.push_str(&format!(
11584 "\n---\nmagi:run/{} magi:candidate-{}\n",
11585 state.id,
11586 winner.to_ascii_lowercase()
11587 ));
11588
11589 PrMessage { title, body }
11590}
11591
11592fn seeded_instruction(state: &RunState) -> String {
11595 let mut inputs = Vec::new();
11596 if let Some(start) = &state.sources.start_from {
11597 inputs.push(format!(
11598 "Start from `{}` at `{}` (including ancestry).",
11599 start.token, start.sha
11600 ));
11601 }
11602 for pick in &state.sources.pick_commits {
11603 inputs.push(format!(
11604 "Pick `{}` at `{}` (single commit).",
11605 pick.token, pick.sha
11606 ));
11607 }
11608 if inputs.is_empty() {
11609 state.instruction.clone()
11610 } else {
11611 format!(
11612 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11613 state.instruction,
11614 inputs.join("\n")
11615 )
11616 }
11617}
11618
11619async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11623 let base = &state.base_branch;
11624 let mut against = base.clone();
11625 if mode == MergeMode::Pr {
11626 let remote = &state.config.merge.remote;
11630 let tracking = format!("{remote}/{base}");
11631 let fetched = git::fetch(repo, remote, base).await;
11632 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11633 against = tracking;
11634 } else {
11635 return false;
11636 }
11637 }
11638 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11639}
11640
11641fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11644 let mut detail = format!(
11645 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11646 nothing to open a pull request for"
11647 );
11648 detail.push_str(
11649 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11650 );
11651 detail
11652}
11653
11654#[derive(Debug, PartialEq, Eq)]
11657enum PrPlan {
11658 Create,
11659 Adopt { url: String, title: String },
11660 Stop(String),
11661}
11662
11663fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11666 match found {
11667 Ok(land::OpenPr::None) => PrPlan::Create,
11668 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11669 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11670 "several open pull requests exist for this branch, not picking one: {}",
11671 urls.join(" ")
11672 )),
11673 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11674 }
11675}
11676
11677pub(crate) async fn gh_pr_create(
11679 cwd: &Path,
11680 base: &str,
11681 head: &str,
11682 title: &str,
11683 body: &str,
11684) -> Result<String> {
11685 let out = tokio::process::Command::new("gh")
11686 .args([
11687 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11688 ])
11689 .current_dir(cwd)
11690 .quiet()
11691 .stdin(std::process::Stdio::null())
11692 .output()
11693 .await
11694 .context("spawn gh")?;
11695 if out.status.success() {
11696 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11697 } else {
11698 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11699 }
11700}
11701
11702async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11704 match std::fs::symlink_metadata(path) {
11705 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11706 git::worktree_remove_clean(repo, path).await?;
11709 return Ok(());
11710 }
11711 Err(e) => return Err(e.into()),
11712 Ok(_) => {}
11713 }
11714 if !git::is_clean(path).await? {
11715 bail!(
11716 "worktree has uncommitted work; refusing removal: {}",
11717 path.display()
11718 );
11719 }
11720 if !git::worktree_remove_clean(repo, path).await? {
11721 bail!("worktree removal refused: {}", path.display());
11722 }
11723 Ok(())
11724}
11725
11726pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11735 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11736 bail!("refusing fold: {reason}");
11737 }
11738 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11739 bail!("candidate preservation incomplete; refusing cleanup");
11740 }
11741 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11745 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11746 let _driver = crate::driver::maintenance(home, &state.id)?;
11747 let mut output_guards = Vec::new();
11748 for entry in std::fs::read_dir(state.worktree_root())
11749 .into_iter()
11750 .flatten()
11751 .flatten()
11752 {
11753 if entry.path().join(".git").is_file() {
11754 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11755 }
11756 }
11757 let repo = state.repo.clone();
11758 let wait = repository_wait(state, None);
11761 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11762 Ok(lease) => lease,
11763 Err(e) => {
11764 state.save_under(home)?;
11765 return Err(e);
11766 }
11767 };
11768 let _source_outputs = if state.status == RunStatus::Merged
11771 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11772 {
11773 verify_merged_fold_sources(state, drop_winner, home).await?
11774 } else {
11775 Vec::new()
11776 };
11777 let root = state.worktree_root();
11778 let winner = state.tally.as_ref().map(|t| t.winner);
11779 let mut removed = Vec::new();
11780
11781 for i in 0..state.candidates.len() {
11782 let c = state.candidates[i].clone();
11783 let is_winner = Some(c.label) == winner;
11784 if is_winner && !drop_winner {
11785 continue;
11786 }
11787 let existed = c.worktree.exists();
11788 remove_preserved_worktree(&repo, &c.worktree).await?;
11791 if existed {
11792 removed.push(c.worktree.to_string_lossy().into_owned());
11793 }
11794 let handed_over = state.released_branches.contains(&c.branch);
11797 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11798 if !git::branch_delete(&repo, &c.branch).await? {
11799 bail!("candidate branch removal refused");
11800 }
11801 removed.push(c.branch.clone());
11802 }
11803 state.candidates[i].folded = true;
11804 }
11805
11806 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11807 let path = name.path();
11808 let keep = !drop_winner
11809 && winner.is_some_and(|w| {
11810 path.file_name()
11811 .is_some_and(|n| n == format!("cand-{w}").as_str())
11812 });
11813 if keep {
11814 continue;
11815 }
11816 remove_preserved_worktree(&repo, &path).await?;
11817 removed.push(path.to_string_lossy().into_owned());
11818 }
11819
11820 remove_if_empty(&root);
11829
11830 if state.enabled_worktree_config && drop_winner {
11831 git::release_worktree_config(&repo).await.ok();
11835 state.enabled_worktree_config = false;
11836 }
11837 state.save_under(home)?;
11838 Ok(removed)
11839}
11840
11841async fn verify_merged_fold_sources(
11842 state: &RunState,
11843 drop_winner: bool,
11844 home: &Path,
11845) -> Result<Vec<std::fs::File>> {
11846 let winner = state.tally.as_ref().map(|t| t.winner);
11847 let mut heads = BTreeSet::new();
11848 for candidate in &state.candidates {
11849 if Some(candidate.label) == winner && !drop_winner {
11850 continue;
11851 }
11852 anyhow::ensure!(
11853 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11854 "candidate recovery source is outside the inventoried run root: {}",
11855 candidate.worktree.display()
11856 );
11857 match std::fs::symlink_metadata(&candidate.worktree) {
11858 Ok(_) => {}
11859 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11860 Err(error) => return Err(error.into()),
11861 }
11862 if !state.released_branches.contains(&candidate.branch)
11863 && git::branch_exists(&state.repo, &candidate.branch).await?
11864 {
11865 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11866 }
11867 }
11868 let mut cargo_guards = Vec::new();
11869 let entries = match std::fs::read_dir(state.worktree_root()) {
11870 Ok(entries) => Some(entries),
11871 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11872 Err(error) => return Err(error.into()),
11873 };
11874 for entry in entries.into_iter().flatten() {
11875 let path = entry?.path();
11876 let metadata = std::fs::symlink_metadata(&path)?;
11877 anyhow::ensure!(
11878 metadata.is_dir() && !metadata.file_type().is_symlink(),
11879 "non-directory or symlink recovery source retained: {}",
11880 path.display()
11881 );
11882 if !drop_winner
11883 && winner.is_some_and(|w| {
11884 path.file_name()
11885 .is_some_and(|name| name == format!("cand-{w}").as_str())
11886 })
11887 {
11888 continue;
11889 }
11890 anyhow::ensure!(
11891 path.join(".git").is_file(),
11892 "unregistered recovery source retained: {}",
11893 path.display()
11894 );
11895 anyhow::ensure!(
11896 git::git(
11897 &path,
11898 &[
11899 "status",
11900 "--porcelain",
11901 "--untracked-files=all",
11902 "--ignored",
11903 "--",
11904 ".",
11905 ":(exclude)target"
11906 ]
11907 )
11908 .await?
11909 .is_empty(),
11910 "source changes or ignored recovery files retained: {}",
11911 path.display()
11912 );
11913 match std::fs::symlink_metadata(path.join("target")) {
11914 Ok(_) => {
11915 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11916 }
11917 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11918 Err(error) => return Err(error.into()),
11919 }
11920 for checkpoint in [
11921 "rebase-merge",
11922 "rebase-apply",
11923 "sequencer",
11924 "CHERRY_PICK_HEAD",
11925 "MERGE_HEAD",
11926 "REBASE_HEAD",
11927 "REVERT_HEAD",
11928 "BISECT_LOG",
11929 "AUTO_MERGE",
11930 "MERGE_MSG",
11931 ] {
11932 let location = git::git(
11933 &path,
11934 &[
11935 "rev-parse",
11936 "--path-format=absolute",
11937 "--git-path",
11938 checkpoint,
11939 ],
11940 )
11941 .await?;
11942 anyhow::ensure!(
11943 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11944 if e.kind() == std::io::ErrorKind::NotFound),
11945 "Git recovery checkpoint retained: {checkpoint}"
11946 );
11947 }
11948 heads.insert(git::rev_parse(&path, "HEAD").await?);
11949 }
11950 crate::fold_evidence::preserve(state, home, &heads).await?;
11951
11952 Ok(cargo_guards)
11953}
11954
11955fn remove_if_empty(dir: &Path) {
11966 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11967 std::fs::remove_dir(dir).ok();
11968 }
11969}
11970
11971pub fn worst_open(state: &RunState) -> Option<Severity> {
11973 state
11974 .reviews
11975 .last()?
11976 .reviews
11977 .iter()
11978 .flat_map(|r| r.findings.iter())
11979 .map(|f| f.severity)
11980 .max()
11981}
11982
11983async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11987 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11988 git::acquire_worktree_config(repo).await?;
11989 state.enabled_worktree_config = true;
11990 }
11991 Ok(())
11992}
11993
11994#[cfg(test)]
11995mod tests {
11996 #[tokio::test]
11997 async fn launcher_base_fetch_requires_remote_success() {
11998 let tmp = tempfile::tempdir().unwrap();
11999 let repo = tmp.path().join("repo");
12000 let remote = tmp.path().join("remote");
12001 std::fs::create_dir(&repo).unwrap();
12002 std::fs::create_dir(&remote).unwrap();
12003 git::git(&remote, &["init", "--bare", "-b", "main"])
12004 .await
12005 .unwrap();
12006 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12007 git::git(
12008 &repo,
12009 &[
12010 "-c",
12011 "user.name=Fixture",
12012 "-c",
12013 "user.email=fixture@example.invalid",
12014 "commit",
12015 "--allow-empty",
12016 "-m",
12017 "fixture",
12018 ],
12019 )
12020 .await
12021 .unwrap();
12022 git::git(
12023 &repo,
12024 &["remote", "add", "origin", remote.to_str().unwrap()],
12025 )
12026 .await
12027 .unwrap();
12028 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12029 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12030 assert_eq!(
12031 resolve_base(&repo, "main", "origin", None)
12032 .await
12033 .unwrap()
12034 .commit,
12035 head
12036 );
12037 #[cfg(unix)]
12038 {
12039 let policy: crate::config::Coordination = toml::from_str(
12040 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12041 ).unwrap();
12042 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12043 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12044 let held = crate::repo_lock::SeatLease::operation_with(
12045 &repo,
12046 Some(&policy),
12047 &crate::repo_lock::WaitPolicy::default(),
12048 )
12049 .await
12050 .unwrap();
12051 assert!(held.is_some());
12052 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12053 tokio::pin!(fetch);
12054 assert!(
12055 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12056 .await
12057 .is_err()
12058 );
12059 assert!(
12060 !repo.join(".git/FETCH_HEAD").exists(),
12061 "no fetch before admission"
12062 );
12063 drop(held);
12064 let fetched = fetch.await.unwrap();
12065 assert_eq!(fetched.commit, head);
12066 assert_eq!(fetched.remote, "origin");
12067 assert_eq!(fetched.branch, "main");
12068 }
12069 git::git(
12071 &repo,
12072 &[
12073 "remote",
12074 "set-url",
12075 "origin",
12076 tmp.path().join("missing").to_str().unwrap(),
12077 ],
12078 )
12079 .await
12080 .unwrap();
12081 let error = resolve_base(&repo, "main", "origin", None)
12082 .await
12083 .unwrap_err()
12084 .to_string();
12085 assert!(error.contains("cannot read origin/main"));
12086 assert!(error.contains("refusing to branch off"));
12087 }
12088
12089 #[test]
12090 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12091 let leaked = vec!["chore(deps): update a crate".to_owned()];
12092 let own = "fix(daemon): apply a chosen action";
12093 assert!(should_retitle("", own, &leaked));
12094 assert!(should_retitle(
12095 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12096 own,
12097 &leaked
12098 ));
12099 assert!(should_retitle(
12100 "chore: land candidate A of run 1",
12101 own,
12102 &leaked
12103 ));
12104 assert!(should_retitle(
12105 "magi: candidate A (uncommitted work)",
12106 own,
12107 &leaked
12108 ));
12109 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12110 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12111 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12112 }
12113
12114 #[test]
12115 fn pr_merge_plan_creates_adopts_or_stops() {
12116 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12117 assert_eq!(
12118 pr_merge_plan(Ok(land::OpenPr::One {
12119 url: "u".into(),
12120 title: "t".into()
12121 })),
12122 PrPlan::Adopt {
12123 url: "u".into(),
12124 title: "t".into()
12125 }
12126 );
12127 let PrPlan::Stop(many) =
12128 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12129 else {
12130 panic!("many must stop");
12131 };
12132 assert!(many.contains('a') && many.contains('b'));
12133 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12134 panic!("a failed lookup must stop");
12135 };
12136 assert!(err.contains("bad token"));
12137 }
12138
12139 use super::*;
12140 use crate::run::GateStatus;
12141 use std::collections::BTreeMap;
12142 use std::time::Duration;
12143
12144 fn conductor() -> AgentSpec {
12145 AgentSpec {
12146 id: "conductor".to_owned(),
12147 kind: crate::config::AgentKind::Command,
12148 model: None,
12149 command: vec!["true".to_owned()],
12150 extra_args: Vec::new(),
12151 env: BTreeMap::new(),
12152 prompt_delivery: None,
12153 max_turn_secs: None,
12154 }
12155 }
12156
12157 fn spec(id: &str) -> AgentSpec {
12158 AgentSpec {
12159 id: id.to_owned(),
12160 kind: crate::config::AgentKind::Command,
12161 model: None,
12162 command: vec!["true".to_owned()],
12163 extra_args: Vec::new(),
12164 env: BTreeMap::new(),
12165 prompt_delivery: None,
12166 max_turn_secs: None,
12167 }
12168 }
12169
12170 fn ids(xs: &[&str]) -> BTreeSet<String> {
12171 xs.iter().map(|s| (*s).to_owned()).collect()
12172 }
12173
12174 #[test]
12175 fn pick_successor_skips_an_agent_another_seat_holds() {
12176 let roster = [spec("a"), spec("b"), spec("c")];
12178 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12179 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12180 }
12181
12182 #[test]
12183 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12184 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12186 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12187 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12188 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12189 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12190 }
12191
12192 #[test]
12193 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12194 let roster = [spec("a"), spec("b")];
12195 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12196 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12197 let carried = ids(&["b"]);
12198 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12199 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12200 }
12201
12202 #[test]
12203 fn pick_successor_returns_none_without_any_untried_successor() {
12204 let roster = [spec("a"), spec("b")];
12205 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12206 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12207 let carried = ids(&["a"]);
12208 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12209 }
12210
12211 #[test]
12212 fn pick_successor_rescue_avoids_another_seats_occupant() {
12213 let roster = [spec("a"), spec("b"), spec("c")];
12215 let carried = ids(&["b", "c"]);
12216 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12217 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12218 }
12219
12220 #[test]
12221 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12222 let roster = [spec("a"), spec("b"), spec("c")];
12223 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12224 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12225 }
12226
12227 #[test]
12228 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12229 let roster = [spec("a"), spec("b"), spec("c")];
12230 let failed = ids(&["a", "b", "c"]);
12231 let mut tried = ids(&["b"]);
12233 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12234 assert_eq!(first.id, "a");
12235 tried.insert(first.id.clone());
12236 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12237 assert_eq!(second.id, "c");
12238 tried.insert(second.id.clone());
12239 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12240 }
12241
12242 #[test]
12243 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12244 let roster = [spec("a"), spec("b")];
12245 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12246 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12247 }
12248
12249 #[test]
12250 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12251 let roster = [spec("a"), spec("b"), spec("c")];
12252 let h = SeatHistory {
12253 failed: ids(&["a"]),
12254 last_ok: Some("b".to_owned()),
12255 last_fail: None,
12256 };
12257 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12258 let gone = SeatHistory {
12260 last_ok: Some("zzz".to_owned()),
12261 ..h.clone()
12262 };
12263 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12264 let failed = SeatHistory {
12265 failed: ids(&["a", "b"]),
12266 last_ok: Some("b".to_owned()),
12267 last_fail: None,
12268 };
12269 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12270 }
12271
12272 #[test]
12273 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12274 let first = SeatState::new("review-1", "alpha", 7);
12275 let next = handover_seat("review-1", "gamma", 7);
12276 assert_ne!(first.claude_session, next.claude_session);
12277 }
12278
12279 #[test]
12280 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12281 let mut state = state_with_summary("x", "y");
12282 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12283 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12284 assert_ne!(a.claude_session, b.claude_session);
12285 }
12286
12287 #[test]
12288 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12289 let roster = [spec("a"), spec("b")];
12290 let h = SeatHistory {
12291 failed: ids(&["a", "b"]),
12292 ..SeatHistory::default()
12293 };
12294 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12295 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12296 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12297 }
12298
12299 #[test]
12306 fn failure_signature_ignores_numbers_and_paths() {
12307 assert_eq!(
12308 failure_signature("exited with Some(2) and no usable output"),
12309 failure_signature("exited with Some(137) and no usable output")
12310 );
12311 assert_eq!(
12312 failure_signature("process exited with no usable output (exit code 2)"),
12313 failure_signature("process exited with no usable output (exit code 137)")
12314 );
12315 for (old, new) in [
12316 (
12317 "exited with Some(2) and no usable output",
12318 "process exited with no usable output (exit code 2)",
12319 ),
12320 (
12321 "exited with None and no usable output",
12322 "process exited with no usable output (exit code unavailable)",
12323 ),
12324 ] {
12325 let previous = FailClass::Other(failure_signature(old));
12326 let current = FailClass::Other(failure_signature(new));
12327 assert_eq!(previous, current, "resumption retains the failure class");
12328 assert!(!should_hand_over(Some(&previous), ¤t));
12329 }
12330 assert_eq!(
12331 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12332 failure_signature("cannot open /var/x.txt: denied")
12333 );
12334 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12335 }
12336
12337 #[test]
12338 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12339 let other = FailClass::Other("x".into());
12340 assert!(should_hand_over(None, &FailClass::Quota));
12341 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12342 assert!(should_hand_over(
12343 Some(&FailClass::Timeout),
12344 &FailClass::Timeout
12345 ));
12346 assert!(should_hand_over(None, &other));
12347 assert!(!should_hand_over(Some(&other), &other));
12348 assert!(should_hand_over(
12349 Some(&other),
12350 &FailClass::Other("y".into())
12351 ));
12352 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12354 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12355 }
12356
12357 #[test]
12358 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12359 let a = SeatState::new("judge-1", "alpha", 7);
12360 let b = handover_seat("judge-1", "beta", 7);
12361 assert_ne!(a.claude_session, b.claude_session);
12362 assert_eq!(b.turns, 0);
12363 }
12364
12365 #[test]
12366 fn a_timeout_is_classified_apart_from_other_failures() {
12367 assert_eq!(
12368 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12369 Some(FailClass::Timeout)
12370 );
12371 assert!(matches!(
12372 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12373 Some(FailClass::Other(_))
12374 ));
12375 }
12376
12377 #[test]
12378 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12379 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12380 let tried = BTreeSet::from(["beta".to_owned()]);
12381 let next = next_untried_in_roster(&roster, 1, &tried);
12384 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12385 }
12386
12387 #[test]
12388 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12389 let roster = vec![spec("alpha"), spec("beta")];
12390 let tried = BTreeSet::from(["beta".to_owned()]);
12391 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12395 }
12396
12397 #[test]
12398 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12399 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12400 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12401 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12405 }
12406
12407 #[test]
12408 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12409 let roster = vec![spec("a"), spec("a"), spec("b")];
12410 let tried = BTreeSet::from(["a".to_owned()]);
12411 let next = next_untried_in_roster(&roster, 0, &tried);
12412 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12413 }
12414
12415 #[test]
12416 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12417 let roster = vec![spec("a"), spec("b")];
12418 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12419 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12420 }
12421
12422 #[test]
12423 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12424 let dir = tempfile::tempdir().unwrap();
12425 let bay = dir.path().join("ffff");
12426
12427 remove_if_empty(&bay);
12429 assert!(!bay.exists());
12430
12431 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12434 remove_if_empty(&bay);
12435 assert!(bay.exists(), "non-empty directory must survive");
12436
12437 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12439 remove_if_empty(&bay);
12440 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12441 }
12442
12443 #[test]
12452 fn a_full_panel_that_found_nothing_is_clean() {
12453 assert!(round_is_clean(
12454 0,
12455 true,
12456 2,
12457 2,
12458 0,
12459 IncompleteReviewPolicy::Block
12460 ));
12461 }
12462
12463 #[test]
12464 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12465 assert!(!round_is_clean(
12466 0,
12467 true,
12468 1,
12469 2,
12470 0,
12471 IncompleteReviewPolicy::Block
12472 ));
12473 }
12474
12475 #[test]
12476 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12477 assert!(!round_is_clean(
12478 1,
12479 true,
12480 1,
12481 2,
12482 0,
12483 IncompleteReviewPolicy::Warn
12484 ));
12485 }
12486
12487 #[test]
12488 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12489 assert!(round_is_clean(
12490 0,
12491 true,
12492 1,
12493 2,
12494 0,
12495 IncompleteReviewPolicy::Warn
12496 ));
12497 }
12498
12499 #[test]
12500 fn a_full_panel_with_an_open_finding_is_not_clean() {
12501 assert!(!round_is_clean(
12502 1,
12503 true,
12504 2,
12505 2,
12506 0,
12507 IncompleteReviewPolicy::Block
12508 ));
12509 }
12510
12511 #[test]
12512 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12513 assert!(!round_is_clean(
12514 0,
12515 false,
12516 2,
12517 2,
12518 0,
12519 IncompleteReviewPolicy::Block
12520 ));
12521 }
12522
12523 #[test]
12530 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12531 assert!(round_is_clean(
12534 0,
12535 true,
12536 1,
12537 2,
12538 1,
12539 IncompleteReviewPolicy::Block
12540 ));
12541 }
12542
12543 #[test]
12544 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12545 assert!(!round_is_clean(
12548 0,
12549 true,
12550 1,
12551 2,
12552 0,
12553 IncompleteReviewPolicy::Block
12554 ));
12555 }
12556
12557 #[test]
12558 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12559 assert!(!round_is_clean(
12560 1,
12561 true,
12562 1,
12563 2,
12564 1,
12565 IncompleteReviewPolicy::Block
12566 ));
12567 assert!(!round_is_clean(
12568 0,
12569 false,
12570 1,
12571 2,
12572 1,
12573 IncompleteReviewPolicy::Block
12574 ));
12575 }
12576
12577 #[test]
12578 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12579 assert!(!round_is_clean(
12583 0,
12584 true,
12585 0,
12586 2,
12587 2,
12588 IncompleteReviewPolicy::Block
12589 ));
12590 }
12591
12592 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12593 CommandOutcome {
12594 command: "test".to_owned(),
12595 code,
12596 output_tail: String::new(),
12597 duration_ms: 0,
12598 resource_blocked,
12599 }
12600 }
12601
12602 #[test]
12603 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12604 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12605 assert!(
12606 !verify_inconclusive(&[outcome(Some(1), false)]),
12607 "an ordinary failure is still evidence about the patch"
12608 );
12609 assert!(verify_inconclusive(&[outcome(None, true)]));
12610 assert!(
12611 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12612 "one inconclusive outcome taints the whole batch"
12613 );
12614 assert!(!verify_inconclusive(&[]));
12615 }
12616
12617 #[tokio::test]
12618 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12619 let calls = std::sync::atomic::AtomicUsize::new(0);
12623 let started = Instant::now();
12624 wait_for_pids_with(
12625 &[123],
12626 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12627 Duration::from_millis(5),
12628 Duration::from_secs(5),
12629 )
12630 .await;
12631 assert!(
12632 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12633 "must keep checking rather than deciding on the first answer"
12634 );
12635 assert!(
12636 started.elapsed() < Duration::from_secs(1),
12637 "must return the moment it is confirmed dead, not wait out the ceiling"
12638 );
12639 }
12640
12641 #[tokio::test]
12642 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12643 let started = Instant::now();
12644 wait_for_pids_with(
12645 &[123],
12646 |_| true, Duration::from_millis(5),
12648 Duration::from_millis(30),
12649 )
12650 .await;
12651 let elapsed = started.elapsed();
12652 assert!(
12653 elapsed >= Duration::from_millis(30),
12654 "must not give up before its own ceiling: {elapsed:?}"
12655 );
12656 assert!(
12657 elapsed < Duration::from_secs(1),
12658 "must not wait past its own ceiling either: {elapsed:?}"
12659 );
12660 }
12661
12662 #[tokio::test]
12663 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12664 let started = Instant::now();
12665 wait_for_pids_with(
12666 &[],
12667 |_| true,
12668 Duration::from_secs(5),
12669 Duration::from_secs(5),
12670 )
12671 .await;
12672 assert!(
12673 started.elapsed() < Duration::from_millis(200),
12674 "an empty pid list has nothing to confirm"
12675 );
12676 }
12677
12678 fn review_round(
12684 clean: bool,
12685 blocking: usize,
12686 answered: usize,
12687 expected: usize,
12688 progressed: bool,
12689 e2e_ok: bool,
12690 ) -> ReviewRound {
12691 ReviewRound {
12692 round: 1,
12693 head: "h".to_owned(),
12694 verified_head: None,
12695 verified_at: None,
12696 reviews: Vec::new(),
12697 e2e: vec![CommandOutcome {
12698 command: "test".to_owned(),
12699 code: Some(if e2e_ok { 0 } else { 1 }),
12700 output_tail: String::new(),
12701 duration_ms: 0,
12702 resource_blocked: false,
12703 }],
12704 verify_retried: false,
12705 e2e_deferred: false,
12706 e2e_defer_reason: None,
12707 fix: None,
12708 blocking,
12709 answered,
12710 expected,
12711 clean,
12712 progressed,
12713 vote_split: false,
12714 reconsideration: Vec::new(),
12715 verdict: None,
12716 }
12717 }
12718
12719 #[test]
12720 fn review_conclusion_is_none_when_nothing_has_run() {
12721 assert_eq!(review_conclusion(&[], 3), None);
12722 }
12723
12724 #[test]
12725 fn review_conclusion_is_none_while_rounds_remain() {
12726 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12727 assert_eq!(review_conclusion(&rounds, 3), None);
12728 }
12729
12730 #[test]
12731 fn review_conclusion_is_gating_once_a_round_is_clean() {
12732 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12733 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12734 }
12735
12736 #[test]
12737 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12738 let rounds = vec![
12739 review_round(false, 1, 2, 2, true, true),
12740 review_round(false, 1, 2, 2, true, true),
12741 ];
12742 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12743 }
12744
12745 #[test]
12746 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12747 let rounds = vec![
12748 review_round(false, 1, 2, 2, true, true),
12749 review_round(false, 1, 2, 2, true, false),
12750 ];
12751 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12752 }
12753
12754 #[test]
12755 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12756 let mut blocked = review_round(false, 1, 2, 2, true, false);
12763 blocked.e2e[0].resource_blocked = true;
12764 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12765 assert_eq!(review_conclusion(&rounds, 2), None);
12766 }
12767
12768 #[test]
12769 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12770 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12772 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12773 }
12774
12775 #[test]
12776 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12777 let rounds = vec![
12778 review_round(false, 1, 2, 2, false, true),
12779 review_round(false, 1, 2, 2, false, true),
12780 ];
12781 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12782 }
12783
12784 fn secs(n: u64) -> Duration {
12785 Duration::from_secs(n)
12786 }
12787
12788 #[tokio::test]
12791 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12792 ask_test_home();
12793 let tmp = tempfile::tempdir().unwrap();
12794 let repo = tmp.path().join("repo");
12795 std::fs::create_dir_all(&repo).unwrap();
12796 init_repo(&repo);
12797 let g = |args: &[&str]| {
12798 let out = std::process::Command::new("git")
12799 .args(args)
12800 .current_dir(&repo)
12801 .quiet()
12802 .output()
12803 .unwrap();
12804 assert!(
12805 out.status.success(),
12806 "{}",
12807 String::from_utf8_lossy(&out.stderr)
12808 );
12809 };
12810 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12811 g(&["checkout", "-b", "historical"]);
12812 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12813 g(&["add", "."]);
12814 g(&["commit", "-m", "old alternative"]);
12815 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12816 g(&["checkout", "-b", "magi/task-selected", &base]);
12817 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12818 g(&["add", "."]);
12819 g(&["commit", "-m", "measurement infrastructure"]);
12820 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12821 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12822 g(&["add", "."]);
12823 g(&["commit", "-m", "dependent implementation"]);
12824 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12825 g(&["checkout", "main"]);
12826 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12827 g(&["add", "."]);
12828 g(&["commit", "-m", "unrelated base change"]);
12829 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12830 let origin = tmp.path().join("origin.git");
12831 g(&[
12832 "clone",
12833 "--bare",
12834 repo.to_str().unwrap(),
12835 origin.to_str().unwrap(),
12836 ]);
12837 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12838 g(&["fetch", "origin"]);
12839 let selection = crate::pr_recovery::Selection {
12840 url: "https://example.test/project/pull/1".into(),
12841 branch: "magi/task-selected".into(),
12842 head: head.clone(),
12843 source_head: None,
12844 source_run: None,
12845 source_proof: None,
12846 disposition: Some(crate::pr_recovery::Disposition::Review),
12847 instruction: "preserve measured behavior".into(),
12848 approved_at: jiff::Timestamp::now(),
12849 };
12850 let mut template = runner_at(RunStatus::Reviewing);
12851 template.state.repo = repo.clone();
12852 template.state.base_commit = current.clone();
12853 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12854 template.state.origin = Some(crate::run::Origin::queue("selected"));
12855 template.state.pr_head = Some(selection.branch.clone());
12856 template.state.pr_recovery = Some(selection.clone());
12857 let mut runner =
12858 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12859 .await
12860 .unwrap();
12861 let wt = runner.state.winner().unwrap().worktree.clone();
12862 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12863 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12864 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12865 assert!(runner.state.seeds.is_empty());
12866 assert_eq!(runner.state.instruction, "preserve measured behavior");
12867 runner.sync_to_base().await.unwrap();
12868 assert_eq!(
12869 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12870 "prerequisite"
12871 );
12872 assert_eq!(
12873 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12874 "measured implementation"
12875 );
12876 assert_eq!(
12877 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12878 "current main change"
12879 );
12880 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12881 assert_eq!(
12882 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12883 head
12884 );
12885
12886 let mut partial = runner_at(RunStatus::Reviewing);
12888 partial.state.repo = repo.clone();
12889 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12890 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12891 run: Some("exit 1".into()),
12892 ..Default::default()
12893 }];
12894 let selection = runner.state.pr_recovery.clone().unwrap();
12895 partial.state.pr_recovery = Some(selection.clone());
12896 partial.state.pr_head = Some(selection.branch.clone());
12897 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12898 let id = partial.state.id.clone();
12899 let error = match Runner::open_selected(
12900 &repo,
12901 partial.state,
12902 partial.roles,
12903 1,
12904 runner.state.base_commit.clone(),
12905 selection,
12906 )
12907 .await
12908 {
12909 Ok(_) => panic!("setup must fail"),
12910 Err(e) => e,
12911 };
12912 assert_eq!(
12913 error
12914 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12915 .unwrap()
12916 .run,
12917 id
12918 );
12919 let failed = RunState::load(&id).unwrap();
12920 assert_eq!(failed.status, RunStatus::Blocked);
12921 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12922 assert!(Runner::resume(&id).is_err());
12923 runner.state.status = RunStatus::Ready;
12924 runner.state.driver_exited = true;
12925 runner.state.save().unwrap();
12926 let mut retry = runner_at(RunStatus::Reviewing);
12927 retry.state.repo = repo.clone();
12928 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12929 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12930 reselected.approved_at = jiff::Timestamp::now();
12931 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12932 retry.state.pr_head = Some(reselected.branch.clone());
12933 retry.state.pr_recovery = Some(reselected.clone());
12934 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12935 .await
12936 .unwrap();
12937 let restarted = Runner::open_selected(
12938 &repo,
12939 retry.state,
12940 retry.roles,
12941 1,
12942 runner.state.base_commit.clone(),
12943 reselected,
12944 )
12945 .await
12946 .unwrap();
12947 assert_eq!(
12948 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12949 .await
12950 .unwrap(),
12951 head
12952 );
12953 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12954 assert_eq!(
12955 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12956 head
12957 );
12958 }
12959
12960 #[tokio::test]
12961 #[cfg(unix)]
12962 async fn fallback_and_interrupted_restoration_require_coordination() {
12963 use std::os::fd::AsRawFd as _;
12964 ask_test_home();
12965 let tmp = tempfile::tempdir().unwrap();
12966 let mut runner = registered_runner(tmp.path()).await;
12967 runner.state.config.graph.timeout_resource_wait = 1;
12968 let winner = runner.state.candidates[0].clone();
12969 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12970 let leftover = winner.worktree.join("leftover");
12971 std::fs::write(&leftover, "untrusted edits").unwrap();
12972 let next = tmp.path().join("next-fixer");
12973 let mut failed = spec("first");
12974 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12975 let mut successor = spec("second");
12976 successor.command = vec![
12977 "sh".into(),
12978 "-c".into(),
12979 format!("touch '{}'; echo done", next.display()),
12980 ];
12981 let attempts = vec![
12982 (failed.clone(), "first".into()),
12983 (successor.clone(), "second".into()),
12984 ];
12985 let lock_path = runner.state.repo.join(".git/updater.lock");
12986 let external = std::fs::File::create(&lock_path).unwrap();
12987 assert_eq!(
12988 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12989 0
12990 );
12991 let artifacts = tmp.path().join("artifacts");
12992 let build = |spec, seat| SeatJob {
12993 spec,
12994 seat,
12995 cwd: winner.worktree.clone(),
12996 prompt: "fixture".into(),
12997 timeout: Duration::from_secs(10),
12998 allow_write: true,
12999 sessions: false,
13000 artifacts: artifacts.clone(),
13001 stem: "fix".into(),
13002 handover: None,
13003 };
13004 let (_, _, out) = runner
13005 .ask_fixer_among(
13006 attempts.clone(),
13007 "graft",
13008 None,
13009 Some((&winner.worktree, &head)),
13010 build,
13011 )
13012 .await;
13013 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13014 assert!(leftover.exists(), "contention must prevent clean");
13015 assert!(!next.exists());
13016 assert!(runner.state.handovers.is_empty());
13017 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13019 from_run: "earlier".into(),
13020 generation: 1,
13021 mode: crate::recovery::Mode::VerifyRepair,
13022 reasons: Vec::new(),
13023 reviewed_head: None,
13024 rounds: Vec::new(),
13025 open_findings: Vec::new(),
13026 evidence: None,
13027 criteria_inherited: true,
13028 repair: crate::recovery::Repair::Started {
13029 before: head.clone(),
13030 },
13031 });
13032 assert!(
13033 runner
13034 .recovery_repair(&winner)
13035 .await
13036 .unwrap_err()
13037 .is::<crate::repo_lock::ResourceBlocked>()
13038 );
13039 assert!(matches!(
13040 runner.state.recovery.as_ref().unwrap().repair,
13041 crate::recovery::Repair::Started { .. }
13042 ));
13043 assert!(leftover.exists());
13044 drop(external);
13045 runner.recovery_repair(&winner).await.unwrap();
13046 assert!(!leftover.exists());
13047 assert!(matches!(
13048 runner.state.recovery.as_ref().unwrap().repair,
13049 crate::recovery::Repair::Done { .. }
13050 ));
13051 std::fs::remove_file(&lock_path).unwrap();
13053 std::fs::create_dir(&lock_path).unwrap();
13054 std::fs::write(&leftover, "untrusted edits").unwrap();
13055 let (_, _, out) = runner
13056 .ask_fixer_among(
13057 attempts,
13058 "graft",
13059 None,
13060 Some((&winner.worktree, &head)),
13061 build,
13062 )
13063 .await;
13064 assert!(matches!(out, AgentOutcome::Failed(_)));
13065 assert!(leftover.exists());
13066 assert!(!next.exists());
13067 assert_eq!(
13068 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13069 head
13070 );
13071 }
13072
13073 #[cfg(unix)]
13074 async fn registered_runner(root: &Path) -> Runner {
13075 ask_test_home();
13076 let repo = root.join("repo");
13077 std::fs::create_dir_all(&repo).unwrap();
13078 init_repo(&repo);
13079 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13080 let mut runner = runner_at(RunStatus::Prep);
13081 runner.state.repo = repo.clone();
13082 runner.state.config.coordination = crate::config::test_coordination();
13083 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13084 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13085 runner.state.config.graph.advise = false;
13086 runner.roles.implementers = vec![spec("implement")];
13087 runner.roles.judges = vec![spec("judge")];
13088 runner.prep().await.unwrap();
13089 runner
13090 }
13091
13092 #[tokio::test]
13093 #[cfg(unix)]
13094 async fn parked_implementer_persists_completed_work() {
13095 for registered in [false, true] {
13096 let tmp = tempfile::tempdir().unwrap();
13097 let mut runner = registered_runner(tmp.path()).await;
13098 if !registered {
13099 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13100 }
13101 let marker = tmp.path().join("started");
13102 let release = tmp.path().join("release");
13103 let mut implementer = spec("implement");
13104 implementer.command = vec![
13105 "sh".into(),
13106 "-c".into(),
13107 format!(
13108 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13109 marker.display(),
13110 release.display()
13111 ),
13112 ];
13113 runner.roles.implementers = vec![implementer.clone()];
13114 runner.state.config.agents = vec![implementer];
13115 let pause = runner.pause.clone();
13116 let (result, ()) = tokio::join!(runner.implement(), async {
13117 tokio::time::timeout(Duration::from_secs(10), async {
13118 while !marker.exists() {
13119 tokio::time::sleep(Duration::from_millis(10)).await;
13120 }
13121 })
13122 .await
13123 .unwrap();
13124 pause.park();
13125 std::fs::write(&release, "go").unwrap();
13126 });
13127 result.unwrap();
13128 let candidate = &runner.state.candidates[0];
13129 assert!(candidate.commits > 0, "{candidate:?}");
13130 assert!(candidate.summary.contains("saved-result"));
13131 assert_eq!(candidate.files, 1);
13132 let before = candidate.commits;
13133 runner.implement().await.unwrap();
13134 assert_eq!(runner.state.candidates[0].commits, before);
13135 }
13136 }
13137
13138 #[tokio::test]
13139 #[cfg(unix)]
13140 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13141 let tmp = tempfile::tempdir().unwrap();
13142 let mut runner = registered_runner(tmp.path()).await;
13143 let winner = runner.state.candidates[0].clone();
13144 let cache = tmp.path().join("cache");
13145 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13146 runner.state.config.graph.timeout_resource_wait = 0;
13147 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13148 let crate::cache::AcquireOutcome::Acquired(held) =
13149 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13150 else {
13151 panic!("cache must be free")
13152 };
13153 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13154 from_run: "earlier".into(),
13155 generation: 1,
13156 mode: crate::recovery::Mode::VerifyRepair,
13157 reasons: Vec::new(),
13158 reviewed_head: None,
13159 rounds: Vec::new(),
13160 open_findings: Vec::new(),
13161 evidence: None,
13162 criteria_inherited: true,
13163 repair: crate::recovery::Repair::NotStarted,
13164 });
13165 runner.recovery_repair(&winner).await.unwrap();
13166 assert!(matches!(
13167 runner.state.recovery.as_ref().unwrap().repair,
13168 crate::recovery::Repair::WaitingForRestore { .. }
13169 ));
13170 drop(held);
13171 runner.state.resource_block = None;
13172 runner.state.config.graph.timeout_resource_wait = 1;
13173 runner.recovery_repair(&winner).await.unwrap();
13174 assert!(matches!(
13175 runner.state.recovery.as_ref().unwrap().repair,
13176 crate::recovery::Repair::Done { .. }
13177 ));
13178 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13179 }
13180
13181 #[tokio::test]
13182 #[cfg(unix)]
13183 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13184 use std::os::fd::AsRawFd as _;
13185 for graft in [false, true] {
13186 let tmp = tempfile::tempdir().unwrap();
13187 let mut runner = registered_runner(tmp.path()).await;
13188 runner.state.config.graph.timeout_resource_wait = 1;
13189 let winner = runner.state.candidates[0].clone();
13190 let marker = tmp.path().join("fixer-started");
13191 let mut fixer = spec("fixer");
13192 fixer.command = vec![
13193 "sh".into(),
13194 "-c".into(),
13195 format!("echo started >> '{}'; echo done", marker.display()),
13196 ];
13197 runner.roles.fixer = Some(vec![fixer]);
13198 if graft {
13199 let mut loser = winner.clone();
13200 loser.label = 'B';
13201 loser.index = 1;
13202 runner.state.candidates.push(loser);
13203 runner.state.config.graph.graft_runner_up = true;
13204 runner.state.tally = Some(Tally {
13205 first_choice: BTreeMap::new(),
13206 borda: BTreeMap::new(),
13207 winner: winner.label,
13208 rankings: 1,
13209 unanimous_initial: true,
13210 deliberated: false,
13211 changed_votes: 0,
13212 unanimous_final: true,
13213 tie_break: None,
13214 judges: 1,
13215 present: 1,
13216 quorum: 1,
13217 met_quorum: true,
13218 uncontested: None,
13219 });
13220 runner.state.judgements.push(crate::run::Judgement {
13221 judge: 1,
13222 seat: "judge-1".into(),
13223 agent: "judge".into(),
13224 ranking: vec![winner.label, 'B'],
13225 reasons: BTreeMap::new(),
13226 confidence: None,
13227 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13228 order: vec![0, 1],
13229 failed: None,
13230 duration_ms: 0,
13231 });
13232 } else {
13233 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13234 from_run: "earlier".into(),
13235 generation: 1,
13236 mode: crate::recovery::Mode::VerifyRepair,
13237 reasons: Vec::new(),
13238 reviewed_head: None,
13239 rounds: Vec::new(),
13240 open_findings: Vec::new(),
13241 evidence: None,
13242 criteria_inherited: true,
13243 repair: crate::recovery::Repair::NotStarted,
13244 });
13245 }
13246 let reader = crate::repo_lock::SeatLease::for_checkout(
13247 &winner.worktree,
13248 false,
13249 crate::config::test_coordination().as_ref(),
13250 )
13251 .await
13252 .unwrap();
13253 let pause = runner.pause.clone();
13254 let (result, ()) = tokio::join!(
13255 async {
13256 if graft {
13257 runner.graft().await
13258 } else {
13259 runner.recovery_repair(&winner).await
13260 }
13261 },
13262 async {
13263 tokio::time::sleep(Duration::from_millis(100)).await;
13264 pause.park();
13265 }
13266 );
13267 result.unwrap();
13268 assert!(!marker.exists(), "a blocked seat must not spawn");
13269 runner.state = RunState::load(&runner.state.id).unwrap();
13271 if graft {
13272 assert_eq!(
13273 runner.state.graft.as_ref().unwrap().outcome,
13274 GraftOutcome::WaitingForRestore
13275 );
13276 } else {
13277 assert!(matches!(
13278 runner.state.recovery.as_ref().unwrap().repair,
13279 crate::recovery::Repair::WaitingForRestore { .. }
13280 ));
13281 }
13282 drop(reader);
13283 runner.pause = Pause::new();
13284 runner.state.resource_block = None;
13285 let external =
13287 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13288 assert_eq!(
13289 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13290 0
13291 );
13292 let leftover = winner.worktree.join("leftover");
13293 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13294 let result = if graft {
13295 runner.graft().await
13296 } else {
13297 runner.recovery_repair(&winner).await
13298 };
13299 assert!(
13300 result
13301 .unwrap_err()
13302 .is::<crate::repo_lock::ResourceBlocked>()
13303 );
13304 assert!(leftover.exists());
13305 assert!(!marker.exists());
13306 drop(external);
13307 runner.state.resource_block = None;
13308 let result = if graft {
13309 runner.graft().await
13310 } else {
13311 runner.recovery_repair(&winner).await
13312 };
13313 result.unwrap();
13314 assert!(!leftover.exists(), "restore must precede retry");
13315 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13316 if graft {
13317 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13318 } else {
13319 assert!(matches!(
13320 runner.state.recovery.as_ref().unwrap().repair,
13321 crate::recovery::Repair::Done { .. }
13322 ));
13323 }
13324 }
13325 }
13326
13327 #[tokio::test]
13328 #[cfg(unix)]
13329 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13330 use std::os::fd::AsRawFd as _;
13331 let tmp = tempfile::tempdir().unwrap();
13332 let mut runner = registered_runner(tmp.path()).await;
13333 runner.state.config.graph.timeout_resource_wait = 1;
13334 let winner = runner.state.candidates[0].clone();
13335 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13336 runner.state.tally = Some(Tally {
13337 first_choice: BTreeMap::new(),
13338 borda: BTreeMap::new(),
13339 winner: winner.label,
13340 rankings: 0,
13341 unanimous_initial: false,
13342 deliberated: false,
13343 changed_votes: 0,
13344 unanimous_final: false,
13345 tie_break: None,
13346 judges: 0,
13347 present: 0,
13348 quorum: 0,
13349 met_quorum: true,
13350 uncontested: Some("fixture".into()),
13351 });
13352 runner.state.graft = Some(GraftRecord {
13353 pre: pre.clone(),
13354 post: None,
13355 lines: None,
13356 limit: None,
13357 agent: None,
13358 outcome: GraftOutcome::Running,
13359 requested: Vec::new(),
13360 adoption: None,
13361 adoption_error: None,
13362 });
13363 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13364 git::commit_all(&winner.worktree, "unfinished graft")
13365 .await
13366 .unwrap();
13367 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13368 let leftover = winner.worktree.join("leftover");
13369 std::fs::write(&leftover, "untrusted edits").unwrap();
13370 let path = runner.state.repo.join(".git/updater.lock");
13371 let external = std::fs::File::create(&path).unwrap();
13372 assert_eq!(
13373 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13374 0
13375 );
13376 assert!(
13377 runner
13378 .graft()
13379 .await
13380 .unwrap_err()
13381 .is::<crate::repo_lock::ResourceBlocked>()
13382 );
13383 assert!(matches!(
13384 runner.state.graft.as_ref().unwrap().outcome,
13385 GraftOutcome::Running
13386 ));
13387 assert_eq!(
13388 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13389 unfinished
13390 );
13391 assert!(leftover.exists());
13392 drop(external);
13393 std::fs::remove_file(&path).unwrap();
13394 std::fs::create_dir(&path).unwrap();
13395 assert!(runner.graft().await.is_err());
13396 assert!(matches!(
13397 runner.state.graft.as_ref().unwrap().outcome,
13398 GraftOutcome::Running
13399 ));
13400 assert_eq!(
13401 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13402 unfinished
13403 );
13404 assert!(leftover.exists());
13405 std::fs::remove_dir(&path).unwrap();
13406 runner.graft().await.unwrap();
13407 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13408 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13409 assert!(!leftover.exists());
13410 }
13411
13412 #[tokio::test]
13413 #[cfg(unix)]
13414 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13415 use std::os::fd::AsRawFd as _;
13416 let tmp = tempfile::tempdir().unwrap();
13417 let mut runner = registered_runner(tmp.path()).await;
13418 runner.state.prep_complete = Some(false);
13419 runner.state.config.graph.timeout_resource_wait = 1;
13420 runner.state.save().unwrap();
13421 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13422 assert_eq!(
13423 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13424 0
13425 );
13426 runner.execute().await.unwrap();
13427 let disk = RunState::load(&runner.state.id).unwrap();
13428 assert!(disk.driver_exited);
13429 assert_eq!(disk.driver_pid, None);
13430 assert_eq!(runner.state.status, RunStatus::Blocked);
13431 assert_eq!(
13432 runner.state.resource_block.as_ref().unwrap().kind,
13433 "repository coordination"
13434 );
13435 assert!(runner.state.prep_failure.is_none());
13436 assert!(runner.state.seats.is_empty());
13437 runner.pause.park();
13438 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13441 .await
13442 .unwrap()
13443 .unwrap();
13444 assert!(runner.state.parked);
13445 assert!(runner.state.prep_failure.is_none());
13446 let disk = RunState::load(&runner.state.id).unwrap();
13447 assert!(disk.driver_exited);
13448 assert_eq!(disk.driver_pid, None);
13449 }
13450
13451 #[tokio::test]
13452 #[cfg(unix)]
13453 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13454 use std::os::fd::AsRawFd as _;
13455 let tmp = tempfile::tempdir().unwrap();
13456 let mut runner = registered_runner(tmp.path()).await;
13457 runner.state.prep_complete = Some(false);
13458 runner.state.config.graph.timeout_resource_wait = 30;
13459 runner.state.save().unwrap();
13460 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13461 assert_eq!(
13462 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13463 0
13464 );
13465 assert!(
13466 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13467 .await
13468 .is_err()
13469 );
13470 let disk = RunState::load(&runner.state.id).unwrap();
13471 assert!(disk.driver_exited);
13472 assert_eq!(disk.driver_pid, None);
13473 assert!(disk.status.resumable());
13474 assert!(runner.state.driver_exited);
13475 assert_eq!(runner.state.driver_pid, None);
13476 runner.state.save().unwrap();
13477 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13478 }
13479
13480 #[tokio::test]
13481 #[cfg(unix)]
13482 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13483 let tmp = tempfile::tempdir().unwrap();
13484 let mut runner = registered_runner(tmp.path()).await;
13485 let winner = runner.state.candidates[0].clone();
13486 let cache = tmp.path().join("cache");
13487 let started = tmp.path().join("pre-gate-started");
13488 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13489 runner.state.config.verify.gate =
13490 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13491 let home = crate::run::home();
13492 let owner =
13493 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13494 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13495 crate::cache::AcquireOutcome::Acquired(g) => g,
13496 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13497 };
13498 let mut gate = Box::pin(runner.run_gate(&winner));
13499 assert!(
13500 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13501 .await
13502 .is_err()
13503 );
13504 assert!(!started.exists(), "pre-gate must wait for the cache too");
13505 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13509 let editor = crate::repo_lock::SeatLease::checkout_with(
13510 &winner.worktree,
13511 true,
13512 crate::config::test_coordination().as_ref(),
13513 &wait,
13514 )
13515 .await
13516 .unwrap();
13517 drop(editor);
13518 guard.release();
13519 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13520 .await
13521 .unwrap()
13522 .unwrap();
13523 drop(gate);
13524 assert!(outcomes.iter().all(CommandOutcome::ok));
13525 assert!(started.exists());
13526 }
13527
13528 #[tokio::test]
13529 #[cfg(unix)]
13530 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13531 use std::os::fd::AsRawFd as _;
13532
13533 ask_test_home();
13534 let tmp = tempfile::tempdir().unwrap();
13535 let repo = tmp.path().join("repo");
13536 std::fs::create_dir_all(&repo).unwrap();
13537 init_repo(&repo);
13538 let common = repo.join(".git");
13539 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13540 let mut runner = runner_at(RunStatus::Prep);
13541 runner.state.repo = repo.clone();
13542 runner.state.config.coordination = crate::config::test_coordination();
13543 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13544 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13545 runner.state.config.graph.advise = false;
13546 runner.roles.implementers = vec![spec("implement")];
13547 runner.roles.judges = vec![spec("judge")];
13548 runner.prep().await.unwrap();
13549 let winner = runner.state.candidates[0].clone();
13550 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13553 let completed = tmp.path().join("seat-completed");
13554 runner.roles.implementers[0].command = vec![
13555 "sh".into(),
13556 "-c".into(),
13557 format!("touch '{}'; echo completed", completed.display()),
13558 ];
13559 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13560 assert_eq!(
13561 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13562 0
13563 );
13564 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13565 let mut implement = Box::pin(runner.implement());
13566 tokio::select! {
13567 result = &mut implement => panic!("rescue must wait: {result:?}"),
13568 result = tokio::time::timeout(Duration::from_secs(10), async {
13569 while !completed.exists() {
13570 tokio::time::sleep(Duration::from_millis(20)).await;
13571 }
13572 }) => result.unwrap(),
13573 }
13574 assert!(
13575 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13576 .await
13577 .is_err()
13578 );
13579 assert_eq!(
13580 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13581 before
13582 );
13583 assert_eq!(
13584 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13585 0
13586 );
13587 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13588 .await
13589 .unwrap()
13590 .unwrap();
13591 drop(implement);
13592 assert_ne!(
13593 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13594 before
13595 );
13596 assert!(git::is_clean(&winner.worktree).await.unwrap());
13597
13598 let started = tmp.path().join("pre-gate-started");
13599 let proceed = tmp.path().join("pre-gate-proceed");
13600 let checked = tmp.path().join("gate-ran");
13601 runner.state.config.verify.pre_gate = vec![format!(
13602 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13603 started.display(),
13604 proceed.display()
13605 )];
13606 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13607 assert_eq!(
13608 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13609 0
13610 );
13611 let mut gate = Box::pin(runner.run_gate(&winner));
13612 assert!(
13613 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13614 .await
13615 .is_err()
13616 );
13617 assert!(!started.exists());
13618 assert!(!checked.exists());
13619 assert!(git::is_clean(&winner.worktree).await.unwrap());
13620 assert_eq!(
13621 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13622 0
13623 );
13624 tokio::select! {
13625 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13626 result = tokio::time::timeout(Duration::from_secs(10), async {
13627 while !started.exists() {
13628 tokio::time::sleep(Duration::from_millis(20)).await;
13629 }
13630 }) => result.unwrap(),
13631 }
13632 assert_eq!(
13633 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13634 -1,
13635 "mechanical edits must hold the real lock"
13636 );
13637 std::fs::write(proceed, "ready").unwrap();
13638 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13639 .await
13640 .unwrap()
13641 .unwrap();
13642 drop(gate);
13643 assert!(outcomes.iter().all(CommandOutcome::ok));
13644 assert!(checked.exists());
13645 assert!(runner.state.pre_gate_commit.is_some());
13646 assert!(git::is_clean(&winner.worktree).await.unwrap());
13647 assert_eq!(
13648 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13649 0,
13650 "gate completion must release coordination"
13651 );
13652 drop(outsider);
13653 std::fs::remove_file(common.join("updater.lock")).unwrap();
13654 std::fs::create_dir(common.join("updater.lock")).unwrap();
13655 std::fs::remove_file(&started).unwrap();
13656 std::fs::remove_file(&checked).unwrap();
13657 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13658 let error = runner.run_gate(&winner).await.unwrap_err();
13659 assert!(
13660 error
13661 .to_string()
13662 .contains("acquire mandatory repository lock")
13663 );
13664 assert!(
13665 !started.exists(),
13666 "failed acquisition must prevent pre-gate"
13667 );
13668 assert!(!checked.exists(), "failed acquisition must prevent gate");
13669 assert!(git::is_clean(&winner.worktree).await.unwrap());
13670 assert_eq!(
13671 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13672 head
13673 );
13674 }
13675
13676 #[tokio::test]
13677 #[cfg(unix)]
13678 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13679 use std::os::fd::AsRawFd as _;
13680 use std::time::Duration;
13681
13682 ask_test_home();
13683 let tmp = tempfile::tempdir().unwrap();
13684 let repo = tmp.path().join("repo");
13685 std::fs::create_dir_all(&repo).unwrap();
13686 init_repo(&repo);
13687 let common = repo.join(".git");
13688 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13689 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13690 assert_eq!(
13691 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13692 0
13693 );
13694 let mut runner = runner_at(RunStatus::Prep);
13695 runner.state.repo = repo.clone();
13696 runner.state.config.coordination = crate::config::test_coordination();
13697 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13698 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13699 runner.state.config.graph.advise = true;
13700 runner.state.config.graph.advisors = 1;
13701 runner.roles.implementers = vec![spec("implement")];
13702 runner.roles.judges = vec![spec("judge")];
13703 let root = runner.state.worktree_root();
13704 let started = tmp.path().join("setup-started");
13705 let proceed = tmp.path().join("setup-proceed");
13706 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13707 run: Some(format!(
13708 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13709 started.display(),
13710 proceed.display()
13711 )),
13712 ..Default::default()
13713 }];
13714 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13715 let config = git::git(&repo, &["config", "--local", "--list"])
13716 .await
13717 .unwrap();
13718 let mut prep = Box::pin(runner.prep());
13719 assert!(
13720 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13721 .await
13722 .is_err()
13723 );
13724 assert!(
13725 !root.exists(),
13726 "contention must prevent even checkout creation"
13727 );
13728 assert!(!started.exists(), "contention must prevent setup");
13729 assert_eq!(
13730 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13731 branches
13732 );
13733 assert_eq!(
13734 git::git(&repo, &["config", "--local", "--list"])
13735 .await
13736 .unwrap(),
13737 config
13738 );
13739 drop(lock);
13740 tokio::select! {
13741 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13742 result = tokio::time::timeout(Duration::from_secs(10), async {
13743 while !started.exists() {
13744 tokio::time::sleep(Duration::from_millis(20)).await;
13745 }
13746 }) => result.unwrap(),
13747 }
13748 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13749 assert_eq!(
13750 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13751 -1
13752 );
13753 assert!(
13754 tokio::time::timeout(
13755 Duration::from_millis(100),
13756 crate::repo_lock::SeatLease::for_checkout(
13757 &repo,
13758 true,
13759 crate::config::test_coordination().as_ref()
13760 )
13761 )
13762 .await
13763 .is_err(),
13764 "preparation must also exclude local editors"
13765 );
13766 std::fs::write(proceed, "ready").unwrap();
13767 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13768 .await
13769 .unwrap()
13770 .unwrap();
13771 drop(prep);
13772 assert_eq!(runner.state.prep_complete, Some(true));
13773 assert!(root.join("judge-1").exists());
13774 assert!(root.join("advisor-1").exists());
13775 assert_eq!(
13776 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13777 0
13778 );
13779 }
13780
13781 #[tokio::test]
13782 #[cfg(unix)]
13783 async fn preparation_lock_failure_prevents_repository_changes() {
13784 ask_test_home();
13785 let tmp = tempfile::tempdir().unwrap();
13786 let repo = tmp.path().join("repo");
13787 std::fs::create_dir_all(&repo).unwrap();
13788 init_repo(&repo);
13789 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13790 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13793 let mut runner = runner_at(RunStatus::Prep);
13794 runner.state.repo = repo.clone();
13795 runner.state.config.coordination = crate::config::test_coordination();
13796 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13797 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13798 runner.roles.implementers = vec![spec("implement")];
13799 let error = runner.prep().await.unwrap_err();
13800 assert!(
13801 error
13802 .to_string()
13803 .contains("acquire mandatory repository lock")
13804 );
13805 assert!(!runner.state.worktree_root().exists());
13806 assert!(runner.state.candidates.is_empty());
13807 assert!(!repo.join(".git/config.worktree").exists());
13808 assert_eq!(
13809 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13810 "* main"
13811 );
13812 }
13813
13814 #[tokio::test]
13815 async fn preparation_records_creation_intent_before_git_can_fail() {
13816 ask_test_home();
13817 let tmp = tempfile::tempdir().unwrap();
13818 let repo = tmp.path().join("repo");
13819 std::fs::create_dir_all(&repo).unwrap();
13820 init_repo(&repo);
13821 let mut runner = runner_at(RunStatus::Prep);
13822 runner.state.repo = repo.clone();
13823 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13824 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13825 runner.state.config.blind.commit_msg_hook = false;
13826 runner.roles.implementers = vec![spec("one")];
13827 let label = blind::assign_labels(1, runner.state.seed)[0];
13828 let branch = runner.state.branch_for(label);
13829 git::git(&repo, &["branch", &branch]).await.unwrap();
13832 assert!(runner.prep().await.is_err());
13833 let saved = RunState::load(&runner.state.id).unwrap();
13834 assert!(saved.candidates.is_empty());
13835 let event = saved
13836 .events
13837 .iter()
13838 .find(|e| e.node == "prep-intent")
13839 .unwrap();
13840 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13841 assert_eq!(intent["branch"], branch);
13842 assert_eq!(intent["start"], runner.state.base_commit);
13843 assert_eq!(
13844 intent["worktree"],
13845 runner
13846 .state
13847 .worktree_root()
13848 .join(format!("cand-{label}"))
13849 .to_string_lossy()
13850 .as_ref()
13851 );
13852 assert_eq!(intent["picks"], serde_json::json!([]));
13853 }
13854
13855 #[tokio::test]
13856 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13857 ask_test_home();
13858 let tmp = tempfile::tempdir().unwrap();
13859 let repo = tmp.path().join("repo");
13860 std::fs::create_dir_all(&repo).unwrap();
13861 init_repo(&repo);
13862 let mut runner = runner_at(RunStatus::Prep);
13863 runner.state.repo = repo.clone();
13864 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13865 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13866 runner.state.sources.pick_commits.push(refs::Input {
13867 token: "missing".into(),
13868 sha: "0".repeat(40),
13869 provenance: refs::Provenance::ExplicitCli,
13870 });
13871 assert!(runner.prep().await.is_err());
13872 assert!(runner.state.candidates.is_empty());
13873 assert!(!runner.state.worktree_root().exists());
13874 }
13875
13876 #[tokio::test]
13877 async fn prose_and_legacy_inference_never_import_into_candidates() {
13878 ask_test_home();
13879 let tmp = tempfile::tempdir().unwrap();
13880 let repo = tmp.path().join("repo");
13881 std::fs::create_dir_all(&repo).unwrap();
13882 init_repo(&repo);
13883 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13884 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13885 .await
13886 .unwrap();
13887 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13888 git::git(&repo, &["add", "."]).await.unwrap();
13889 git::git(&repo, &["commit", "-m", "historical implementation"])
13890 .await
13891 .unwrap();
13892 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13893 git::git(&repo, &["checkout", "main"]).await.unwrap();
13894 for instruction in [
13895 format!("Example: magi/history/A at {source}"),
13896 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13897 format!("Never import magi/history/A or {source}"),
13898 format!("Conversation answer: I mentioned {source} only as background"),
13899 ] {
13900 let mut runner = runner_at(RunStatus::Prep);
13901 runner.state.repo = repo.clone();
13902 runner.state.base_commit = base.clone();
13903 runner.state.instruction = instruction.clone();
13904 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13905 runner.state.config.blind.commit_msg_hook = false;
13906 runner.state.seeds = vec![refs::Seed {
13907 token: "magi/history/A".into(),
13908 sha: source.clone(),
13909 kind: refs::SeedKind::Unmerged,
13910 branch: true,
13911 detail: String::new(),
13912 }];
13913 runner.roles.implementers = vec![spec("one")];
13914 assert_eq!(seeded_instruction(&runner.state), instruction);
13915 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13917 legacy.as_object_mut().unwrap().remove("sources");
13918 runner.state = serde_json::from_value(legacy).unwrap();
13919 assert_eq!(runner.state.sources, refs::Sources::default());
13920 runner.state.save().unwrap();
13921 runner.state = RunState::load(&runner.state.id).unwrap();
13922 runner.prep().await.unwrap();
13923 for candidate in &runner.state.candidates {
13924 assert_eq!(
13925 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13926 base
13927 );
13928 assert!(!candidate.worktree.join("unwanted").exists());
13929 }
13930 }
13931 }
13932
13933 #[tokio::test]
13934 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13935 ask_test_home();
13936 let tmp = tempfile::tempdir().unwrap();
13937 let repo = tmp.path().join("repo");
13938 std::fs::create_dir_all(&repo).unwrap();
13939 init_repo(&repo);
13940 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13941 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13942 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13943 git::git(&repo, &["add", "."]).await.unwrap();
13944 git::git(&repo, &["commit", "-m", "required seed"])
13945 .await
13946 .unwrap();
13947 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13948 git::git(&repo, &["checkout", "main"]).await.unwrap();
13949 let mut runner = runner_at(RunStatus::Prep);
13950 runner.state.repo = repo.clone();
13951 runner.state.base_commit = base.clone();
13952 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13953 runner.state.config.graph.advise = false;
13954 runner.state.config.blind.commit_msg_hook = false;
13955 runner.state.sources =
13956 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13957 .await
13958 .unwrap();
13959 runner.roles.implementers = vec![spec("one"), spec("two")];
13960 runner.roles.judges = vec![spec("judge")];
13961 let label = blind::assign_labels(2, runner.state.seed)[0];
13962 let branch = runner.state.branch_for(label);
13963 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13964 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13965 .await
13966 .unwrap();
13967 runner.state.candidates.push(Candidate {
13968 index: 0,
13969 label,
13970 agent: "one".into(),
13971 branch,
13972 worktree,
13973 summary: String::new(),
13974 stat: String::new(),
13975 files: 0,
13976 commits: 0,
13977 empty: false,
13978 failed: None,
13979 verified_noop: None,
13980 duration_ms: 0,
13981 folded: false,
13982 });
13983 runner.state.save().unwrap();
13984 runner.state = RunState::load(&runner.state.id).unwrap();
13986 assert_eq!(runner.state.prep_complete, Some(false));
13987 runner.prep().await.unwrap();
13988 assert_eq!(runner.state.prep_complete, Some(true));
13989 assert_eq!(runner.state.candidates.len(), 2);
13990 for candidate in &runner.state.candidates {
13991 assert_eq!(
13992 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13993 git::tree_of(&repo, &seed).await.unwrap()
13994 );
13995 assert_eq!(
13996 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13997 "required commit"
13998 );
13999 }
14000 assert!(runner.state.worktree_root().join("judge-1").exists());
14001 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14002 runner.prep().await.unwrap();
14003 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14004
14005 runner.state.prep_complete = Some(false);
14006 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14007 run: Some(format!(
14008 "test -f '{}'",
14009 tmp.path().join("dependency-ready").display()
14010 )),
14011 ..Default::default()
14012 }];
14013 assert!(runner.prep().await.is_err());
14014 assert!(runner.state.prep_failure.is_some());
14015 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14016 runner.state = RunState::load(&runner.state.id).unwrap();
14017 assert!(
14018 runner.prep().await.is_err(),
14019 "correcting the cause alone must not auto-retry"
14020 );
14021 let failed_at = runner.state.events.last().unwrap().at;
14022 assert!(
14023 runner.retry_preparation(failed_at).is_err(),
14024 "old approvals cannot unlock the hold"
14025 );
14026 let mut task = crate::queue::Task::new(
14027 "retry".into(),
14028 "retry setup".into(),
14029 repo.clone(),
14030 crate::queue::Source::Human,
14031 );
14032 task.start(runner.state.id.clone());
14033 task.hold_machine(Some("setup failed".into()));
14034 task.release_operator("cli", None);
14035 runner
14036 .retry_preparation(task.resumption.as_ref().unwrap().at)
14037 .unwrap();
14038 runner.prep().await.unwrap();
14039 assert!(runner.state.prep_failure.is_none());
14040 assert_eq!(runner.state.prep_complete, Some(true));
14041 assert_eq!(runner.state.candidates.len(), 2);
14042 assert!(runner.state.worktree_root().join("judge-1").exists());
14043 for candidate in &runner.state.candidates {
14044 assert_eq!(
14045 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14046 git::tree_of(&repo, &seed).await.unwrap()
14047 );
14048 }
14049 }
14050
14051 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14052 let mut heads = Vec::new();
14053 for candidate in candidates {
14054 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14055 }
14056 heads
14057 }
14058
14059 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14060 ask_test_home();
14061 let tmp = tempfile::tempdir().unwrap();
14062 let repo = tmp.path().join("repo");
14063 std::fs::create_dir(&repo).unwrap();
14064 init_repo(&repo);
14065 let mut runner = runner_at(RunStatus::Reviewing);
14066 runner.state.repo = repo.clone();
14067 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14068 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14069 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14070 let branch = format!("candidate-{label}");
14071 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14072 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14073 .await
14074 .unwrap();
14075 if label == 'B' {
14076 std::fs::write(worktree.join("change"), "winner").unwrap();
14077 git::rescue_commit(&worktree, "candidate change")
14078 .await
14079 .unwrap();
14080 }
14081 runner.state.candidates.push(Candidate {
14082 index,
14083 label,
14084 agent: "alpha".into(),
14085 branch,
14086 worktree,
14087 summary: "completed reply".into(),
14088 stat: String::new(),
14089 files: usize::from(label == 'B'),
14090 commits: usize::from(label == 'B'),
14091 empty: label == 'A',
14092 failed: None,
14093 verified_noop: None,
14094 duration_ms: 1,
14095 folded: false,
14096 });
14097 }
14098 runner.state.judge_skipped = true;
14099 runner.state.tally = Some(Tally {
14100 first_choice: BTreeMap::new(),
14101 borda: BTreeMap::new(),
14102 winner: 'B',
14103 rankings: 0,
14104 unanimous_initial: false,
14105 deliberated: false,
14106 changed_votes: 0,
14107 unanimous_final: false,
14108 tie_break: None,
14109 judges: 0,
14110 present: 0,
14111 quorum: 0,
14112 met_quorum: true,
14113 uncontested: Some("only B produced a change".into()),
14114 });
14115 runner.state.save().unwrap();
14116 (tmp, runner)
14117 }
14118
14119 #[tokio::test]
14120 #[cfg(unix)]
14121 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14122 for (solo, failed_checkpoint) in
14123 [(true, false), (false, false), (true, true), (false, true)]
14124 {
14125 let (_tmp, mut runner) = preservation_resume_fixture().await;
14126 if solo {
14127 runner.state.candidates.retain(|c| c.label == 'A');
14128 }
14129 runner.state.candidates[0].empty = false;
14130 runner.state.candidates[0].summary.clear();
14131 runner.state.tally = None;
14132 runner.state.judge_skipped = false;
14133 runner.state.status = RunStatus::Implementing;
14134 runner.state.config.disk.min_free_bytes = u64::MAX;
14135 let mut implementer = spec("alpha");
14136 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14137 runner.roles.implementers = vec![implementer];
14138 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14139 if failed_checkpoint {
14140 std::fs::create_dir_all(&patch).unwrap();
14141 }
14142 runner.state.save().unwrap();
14143 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14144 runner.state = RunState::load(&runner.state.id).unwrap();
14145 assert_eq!(runner.state.status, RunStatus::Blocked);
14146 assert!(runner.state.interrupted_implementations.contains(&'A'));
14147 assert_eq!(runner.state.candidates[0].commits, 0);
14148 assert!(!runner.state.candidates[0].empty);
14149 assert!(runner.state.candidates[0].failed.is_none());
14150 if failed_checkpoint {
14151 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14152 std::fs::remove_dir(patch).unwrap();
14153 }
14154 runner.state.resource_block = None;
14155 runner.state.config.disk.min_free_bytes = 0;
14156 runner.roles.implementers[0].command = vec![
14157 "sh".into(),
14158 "-c".into(),
14159 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14160 ];
14161 runner.implement().await.unwrap();
14162 assert!(runner.state.interrupted_implementations.is_empty());
14163 assert!(runner.state.candidate_preservation.is_empty());
14164 assert_eq!(runner.state.candidates[0].commits, 1);
14165 assert!(!runner.state.candidates[0].empty);
14166 assert!(
14167 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14168 .unwrap()
14169 .contains("retried empty interrupted seat")
14170 );
14171 if !solo {
14172 assert_eq!(runner.state.candidates[1].commits, 1);
14173 }
14174 }
14175 }
14176
14177 #[tokio::test]
14178 #[cfg(unix)]
14179 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14180 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14181 let (_tmp, mut runner) = preservation_resume_fixture().await;
14182 runner.state.candidates.retain(|c| c.label == 'A');
14183 runner.state.candidates[0].empty = false;
14184 runner.state.candidates[0].summary.clear();
14185 runner.state.tally = None;
14186 runner.state.judge_skipped = false;
14187 runner.state.status = RunStatus::Implementing;
14188 runner.state.config.disk.min_free_bytes = u64::MAX;
14191 let wt = runner.state.candidates[0].worktree.clone();
14192 std::fs::write(
14193 wt.join("interrupted-source"),
14194 "retain implementation work\n",
14195 )
14196 .unwrap();
14197 if committed {
14198 git::rescue_commit(&wt, "implementation checkpoint")
14199 .await
14200 .unwrap();
14201 }
14202 let lock = git::git(
14203 &wt,
14204 &[
14205 "rev-parse",
14206 "--path-format=absolute",
14207 "--git-path",
14208 "index.lock",
14209 ],
14210 )
14211 .await
14212 .unwrap();
14213 if block_rescue {
14214 std::fs::write(&lock, "fixture-owned lock").unwrap();
14215 }
14216 let mut implementer = spec("alpha");
14217 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14218 runner.roles.implementers = vec![implementer];
14219 runner.state.save().unwrap();
14220 let result = runner.implement().await;
14221 assert_eq!(result.is_err(), block_rescue);
14222 assert_eq!(
14223 runner.state.resource_block.as_ref().unwrap().kind,
14224 "disk pressure"
14225 );
14226 assert!(
14227 runner
14228 .state
14229 .events
14230 .iter()
14231 .any(|e| e.message.contains("agent execution interrupted"))
14232 );
14233 assert!(
14234 runner
14235 .state
14236 .dir()
14237 .join("artifacts/candidate-A-recovery/staged.patch")
14238 .exists()
14239 );
14240 if block_rescue {
14241 let saved = RunState::load(&runner.state.id).unwrap();
14242 assert!(saved.candidate_preservation.contains_key(&'A'));
14243 assert!(!saved.candidates[0].empty);
14244 assert_eq!(
14245 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14246 "retain implementation work\n"
14247 );
14248 std::fs::remove_file(lock).unwrap();
14249 runner.state = saved;
14250 } else {
14251 assert_eq!(runner.state.status, RunStatus::Blocked);
14252 assert_eq!(runner.state.candidates[0].commits, 1);
14253 }
14254 runner.roles.implementers.clear();
14257 runner.state.config.disk.min_free_bytes = 0;
14258 runner.state.resource_block = None;
14259 runner.implement().await.unwrap();
14260 assert!(runner.state.candidate_preservation.is_empty());
14261 assert_eq!(runner.state.candidates[0].commits, 1);
14262 assert!(!runner.state.candidates[0].empty);
14263 assert!(git::is_clean(&wt).await.unwrap());
14264 assert!(
14265 crate::run::read_artifact(&runner.state, "cand-A.patch")
14266 .unwrap()
14267 .contains("retain implementation work")
14268 );
14269 }
14270 }
14271
14272 #[tokio::test]
14273 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14274 let (_tmp, mut runner) = preservation_resume_fixture().await;
14275 runner.implement().await.unwrap();
14277 assert!(runner.state.candidate_preservation.is_empty());
14278 assert!(
14279 !runner
14280 .state
14281 .dir()
14282 .join("artifacts/candidate-A-recovery")
14283 .exists()
14284 );
14285 runner.fold_losers().await.unwrap();
14286 assert!(!runner.state.candidates[0].worktree.exists());
14287 for schema in [14, crate::run::SCHEMA] {
14288 runner.state.schema = schema;
14289 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14290 runner.state.save().unwrap();
14291 runner.state = RunState::load(&runner.state.id).unwrap();
14292 runner.implement().await.unwrap();
14293 assert!(runner.state.candidate_preservation.is_empty());
14294 assert!(runner.state.candidates[0].folded);
14295 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14296 }
14297 }
14298
14299 #[tokio::test]
14300 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14301 let (_tmp, mut runner) = preservation_resume_fixture().await;
14302 let wt = runner.state.candidates[0].worktree.clone();
14303 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14304 runner.state.schema = 14;
14305 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14306 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14307 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14308 for _ in 0..2 {
14309 assert!(runner.implement().await.is_err());
14310 assert_eq!(runner.state.status, RunStatus::Blocked);
14311 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14312 assert!(
14313 runner.state.candidates[0].empty,
14314 "old verdict must remain unusable"
14315 );
14316 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14317 assert!(runner.state.judge_skipped);
14318 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14319 assert!(
14320 crate::run::read_artifact(&runner.state, "cand-A.patch")
14321 .unwrap()
14322 .contains("owned edits")
14323 );
14324 assert!(wt.exists());
14325 assert_eq!(
14326 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14327 .await
14328 .unwrap(),
14329 1
14330 );
14331 runner.state = RunState::load(&runner.state.id).unwrap();
14332 }
14333 }
14334
14335 #[tokio::test]
14336 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14337 let (_tmp, mut runner) = preservation_resume_fixture().await;
14338 let wt = runner.state.candidates[0].worktree.clone();
14339 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14340 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14341 let index_lock = PathBuf::from(
14342 git::git(
14343 &wt,
14344 &[
14345 "rev-parse",
14346 "--path-format=absolute",
14347 "--git-path",
14348 "index.lock",
14349 ],
14350 )
14351 .await
14352 .unwrap(),
14353 );
14354 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14355 runner.state.schema = 14;
14356 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14357 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14358 assert!(runner.implement().await.is_err());
14359 assert!(runner.after_implement().is_err());
14360 assert!(runner.fold_losers().await.is_err());
14361 assert_eq!(
14362 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14363 "failed seat edits"
14364 );
14365 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14366 runner.state = RunState::load(&runner.state.id).unwrap();
14367 std::fs::remove_file(index_lock).unwrap();
14368 assert!(runner.implement().await.is_err());
14370 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14371 assert!(runner.state.candidates[0].empty);
14372 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14373 assert!(
14374 crate::run::read_artifact(&runner.state, "cand-A.patch")
14375 .unwrap()
14376 .contains("failed seat edits")
14377 );
14378 assert!(wt.exists());
14379 }
14380
14381 #[tokio::test]
14382 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14383 let (_tmp, mut runner) = preservation_resume_fixture().await;
14384 let repo = runner.state.repo.clone();
14385 let common = git::git(
14386 &repo,
14387 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14388 )
14389 .await
14390 .unwrap();
14391 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14392 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14393 assert!(runner.fold_losers().await.is_err());
14394 assert!(!runner.state.candidates[0].worktree.exists());
14395 assert!(!runner.state.candidates[0].folded);
14396 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14397 runner.state = RunState::load(&runner.state.id).unwrap();
14398 std::fs::remove_file(branch_lock).unwrap();
14399 let judge = runner.state.worktree_root().join("judge-1");
14400 git::worktree_add_detached(&repo, &judge, "main")
14401 .await
14402 .unwrap();
14403 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14404 runner.roles.judges = vec![spec("judge")];
14405 assert!(runner.fold_losers().await.is_err());
14406 runner.state = RunState::load(&runner.state.id).unwrap();
14407 assert!(
14408 runner.state.candidates[0].folded,
14409 "candidate progress saved before judge failure"
14410 );
14411 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14412 assert!(judge.join("uncommitted").exists());
14413 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14414 runner.fold_losers().await.unwrap();
14415 assert!(!judge.exists());
14416 runner.fold_losers().await.unwrap();
14417 }
14418
14419 #[tokio::test]
14420 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14421 let (_tmp, mut runner) = preservation_resume_fixture().await;
14422 runner.state.candidates.retain(|c| c.label == 'B');
14423 runner.state.tally = None;
14424 runner.state.judge_skipped = false;
14425 runner.state.prep_complete = Some(true);
14426 runner.state.config.graph.advise = false;
14427 runner.state.config.graph.selected_implementer = Some("alpha".into());
14428 let wt = runner.state.candidates[0].worktree.clone();
14429 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14430 runner
14431 .state
14432 .candidate_preservation
14433 .insert('B', "awaiting persistence".into());
14434 let index_lock = git::git(
14435 &wt,
14436 &[
14437 "rev-parse",
14438 "--path-format=absolute",
14439 "--git-path",
14440 "index.lock",
14441 ],
14442 )
14443 .await
14444 .unwrap();
14445 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14446 runner.state.save().unwrap();
14447 assert!(runner.execute_graph().await.is_err());
14448 assert_eq!(runner.state.status, RunStatus::Blocked);
14449 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14450 std::fs::remove_file(index_lock).unwrap();
14451 for _ in 0..2 {
14454 runner.state = RunState::load(&runner.state.id).unwrap();
14455 runner.pause.park();
14456 runner.execute_graph().await.unwrap();
14457 assert_eq!(runner.state.status, RunStatus::Implementing);
14458 assert!(runner.state.candidate_preservation.is_empty());
14459 assert_eq!(runner.state.candidates[0].commits, 2);
14460 assert!(runner.state.tally.is_none());
14461 }
14462 runner.implement().await.unwrap();
14463 assert_eq!(runner.state.status, RunStatus::Judging);
14464 runner.state.status = RunStatus::Blocked;
14466 runner.state.candidates[0].failed = Some("implementation failed".into());
14467 runner.state.save().unwrap();
14468 let error = runner.execute_graph().await.unwrap_err();
14469 assert!(
14470 error
14471 .to_string()
14472 .contains("selected implementation is held")
14473 );
14474 }
14475
14476 #[tokio::test]
14477 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14478 let (_tmp, mut runner) = preservation_resume_fixture().await;
14479 let repo = runner.state.repo.clone();
14480 let candidate = runner.state.candidates[0].clone();
14481 let registered = git::worktree_holding(&repo, &candidate.branch)
14484 .await
14485 .unwrap()
14486 .unwrap();
14487 assert_eq!(
14488 registered.canonicalize().unwrap(),
14489 candidate.worktree.canonicalize().unwrap()
14490 );
14491 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14492 assert_eq!(
14493 git::worktree_holding(&repo, &candidate.branch)
14494 .await
14495 .unwrap(),
14496 Some(registered)
14497 );
14498 assert!(
14499 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14500 "the stale registration initially keeps the branch checked out"
14501 );
14502 runner.fold_losers().await.unwrap();
14503 assert!(runner.state.candidates[0].folded);
14504 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14505 assert_eq!(
14506 git::worktree_holding(&repo, &candidate.branch)
14507 .await
14508 .unwrap(),
14509 None
14510 );
14511 runner.state = RunState::load(&runner.state.id).unwrap();
14512 runner.fold_losers().await.unwrap();
14513 }
14514
14515 #[tokio::test]
14516 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14517 let (_tmp, mut runner) = preservation_resume_fixture().await;
14518 let repo = runner.state.repo.clone();
14519 let candidate = runner.state.candidates[0].clone();
14520 let winner = runner.state.candidates[1].clone();
14521 let registered = git::worktree_holding(&repo, &candidate.branch)
14522 .await
14523 .unwrap()
14524 .unwrap();
14525 assert_eq!(
14526 registered.canonicalize().unwrap(),
14527 candidate.worktree.canonicalize().unwrap()
14528 );
14529 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14530 assert_eq!(
14531 git::worktree_holding(&repo, &candidate.branch)
14532 .await
14533 .unwrap(),
14534 Some(registered)
14535 );
14536 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14537
14538 for _ in 0..2 {
14539 fold_run(&mut runner.state, false, &crate::run::home())
14540 .await
14541 .unwrap();
14542 runner.state = RunState::load(&runner.state.id).unwrap();
14543 assert!(runner.state.candidates[0].folded);
14544 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14545 assert_eq!(
14546 git::worktree_holding(&repo, &candidate.branch)
14547 .await
14548 .unwrap(),
14549 None
14550 );
14551 assert!(!runner.state.candidates[1].folded);
14552 assert!(winner.worktree.exists());
14553 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14554 }
14555 }
14556
14557 #[tokio::test]
14558 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14559 ask_test_home();
14560 let tmp = tempfile::tempdir().unwrap();
14561 let repo = tmp.path().join("repo");
14562 std::fs::create_dir(&repo).unwrap();
14563 init_repo(&repo);
14564 let mut runner = runner_at(RunStatus::Implementing);
14565 runner.state.repo = repo.clone();
14566 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14567 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14568 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14569 let branch = format!("candidate-{label}");
14570 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14571 git::worktree_add_branch(&repo, &wt, &branch, "main")
14572 .await
14573 .unwrap();
14574 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14575 git::git(&wt, &["add", "README.md"]).await.unwrap();
14576 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14577 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14578 runner.state.candidates.push(Candidate {
14579 index,
14580 label,
14581 agent: "alpha".to_owned(),
14582 branch,
14583 worktree: wt,
14584 summary: "completed implementation".to_owned(),
14585 stat: String::new(),
14586 files: 0,
14587 commits: 0,
14588 empty: false,
14589 failed: None,
14590 verified_noop: None,
14591 duration_ms: 42,
14592 folded: false,
14593 });
14594 runner
14595 .state
14596 .candidate_preservation
14597 .insert(label, "awaiting persistence".to_owned());
14598 }
14599 let wt = runner.state.candidates[0].worktree.clone();
14600 let index_lock = git::git(
14601 &wt,
14602 &[
14603 "rev-parse",
14604 "--path-format=absolute",
14605 "--git-path",
14606 "index.lock",
14607 ],
14608 )
14609 .await
14610 .unwrap();
14611 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14612 runner.state.save().unwrap();
14613 assert!(runner.persist_candidates().await.is_err());
14614 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14615 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14616 assert_eq!(
14617 runner.state.candidates[1].commits, 1,
14618 "other results are still collected"
14619 );
14620 assert!(
14621 !runner.state.candidates[0].empty,
14622 "failed rescue is not no change"
14623 );
14624 assert_eq!(
14625 runner.state.candidates[0].summary,
14626 "completed implementation"
14627 );
14628 assert!(runner.after_implement().is_err());
14629 let home = tmp.path().join("fold-home");
14630 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14631 assert!(wt.join("new-file").exists());
14632 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14633 assert!(recovery.join("staged.patch").exists());
14634 assert!(recovery.join("unstaged.patch").exists());
14635 assert_eq!(
14636 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14637 "untracked edits\n"
14638 );
14639 runner.state.begin_pass();
14640 runner.state.end_pass_parked(ParkCause::Upgrade {
14641 from: "1.0".into(),
14642 to: Some("1.1".into()),
14643 });
14644 runner.state.save().unwrap();
14645 runner.state = RunState::load(&runner.state.id).unwrap();
14646 assert!(matches!(
14647 runner.state.passes.last().unwrap().end,
14648 Some(crate::run::PassEnd::Parked {
14649 cause: ParkCause::Upgrade { .. }
14650 })
14651 ));
14652 std::fs::remove_file(index_lock).unwrap();
14653 runner.implement().await.unwrap();
14655 assert!(runner.state.candidate_preservation.is_empty());
14656 assert!(
14657 runner
14658 .state
14659 .candidates
14660 .iter()
14661 .all(|c| c.commits == 1 && !c.empty)
14662 );
14663 assert!(git::is_clean(&wt).await.unwrap());
14664 assert!(
14665 crate::run::read_artifact(&runner.state, "cand-A.patch")
14666 .unwrap()
14667 .contains("later tracked edits")
14668 );
14669 }
14670
14671 #[tokio::test]
14672 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14673 ask_test_home();
14674 let tmp = tempfile::tempdir().unwrap();
14675 let repo = tmp.path().join("repo");
14676 std::fs::create_dir(&repo).unwrap();
14677 init_repo(&repo);
14678 let wt = tmp.path().join("candidate");
14679 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14680 .await
14681 .unwrap();
14682 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14683 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14684 assert!(wt.join("uncommitted").exists());
14685 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14686 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14687 let mut runner = runner_at(RunStatus::Implementing);
14688 runner.state.repo = repo.clone();
14689 runner.state.base_commit = "missing-base".to_owned();
14690 runner.state.candidates.push(Candidate {
14691 index: 0,
14692 label: 'A',
14693 agent: "alpha".into(),
14694 branch: "candidate".into(),
14695 worktree: wt.clone(),
14696 summary: "completed".into(),
14697 stat: String::new(),
14698 files: 0,
14699 commits: 0,
14700 empty: false,
14701 failed: None,
14702 verified_noop: None,
14703 duration_ms: 1,
14704 folded: false,
14705 });
14706 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14707 assert!(
14708 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14709 .await
14710 .is_err()
14711 );
14712 assert!(!runner.state.candidates[0].folded);
14713 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14714 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14715 runner
14716 .state
14717 .candidate_preservation
14718 .insert('A', "pending".into());
14719 assert!(runner.persist_candidates().await.is_err());
14720 assert!(!runner.state.candidates[0].empty);
14721 assert!(!runner.state.candidates[0].folded);
14722 assert!(wt.exists());
14723 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14724 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14725 std::fs::create_dir_all(&patch_path).unwrap();
14726 assert!(
14727 runner.persist_candidates().await.is_err(),
14728 "artifact failure stays pending"
14729 );
14730 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14731 assert!(!runner.state.candidates[0].empty);
14732 std::fs::remove_dir(patch_path).unwrap();
14733 runner.persist_candidates().await.unwrap();
14734 assert!(
14735 runner.state.candidates[0].empty,
14736 "only successful inspection confirms no change"
14737 );
14738 }
14739
14740 fn init_repo(dir: &Path) {
14741 let run = |args: &[&str]| {
14742 let out = std::process::Command::new("git")
14743 .args(args)
14744 .current_dir(dir)
14745 .quiet()
14746 .output()
14747 .expect("spawn git");
14748 assert!(
14749 out.status.success(),
14750 "git {args:?} failed: {}",
14751 String::from_utf8_lossy(&out.stderr)
14752 );
14753 };
14754 run(&["init", "-b", "main"]);
14755 run(&["config", "user.name", "magi test"]);
14756 run(&["config", "user.email", "magi@example.com"]);
14757 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14758 run(&["add", "-A"]);
14759 run(&["commit", "-m", "init"]);
14760 }
14761
14762 fn ask_test_home() {
14770 crate::run::pin_test_home();
14771 }
14772
14773 fn runner_at(status: RunStatus) -> Runner {
14776 let mut state = RunState::new(
14777 PathBuf::from("/nonexistent/repo"),
14778 "main".to_owned(),
14779 "deadbeef".to_owned(),
14780 "task".to_owned(),
14781 Config::default(),
14782 );
14783 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14786 state.id = format!(
14787 "20000101-000000-{:04x}",
14788 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14789 );
14790 state.status = status;
14791 Runner {
14792 state,
14793 roles: ResolvedRoles {
14794 implementers: Vec::new(),
14795 judges: Vec::new(),
14796 reviewers: Vec::new(),
14797 fixer: None,
14798 conductor: conductor(),
14799 implementer_roster: Vec::new(),
14800 judge_roster: Vec::new(),
14801 reviewer_roster: Vec::new(),
14802 },
14803 sem: Arc::new(Semaphore::new(1)),
14804 pause: Pause::new(),
14805 interrupt: Pause::new(),
14806 }
14807 }
14808
14809 #[tokio::test]
14810 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14811 crate::run::pin_test_home();
14812 let mut runner = runner_at(RunStatus::Gating);
14813 runner.state.save().unwrap();
14814 let id = runner.state.id.clone();
14815 let passes = runner.state.passes.len();
14816 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14817 let mut claiming = Box::pin(runner.claim_driver());
14818 std::future::poll_fn(|cx| {
14819 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14820 std::task::Poll::Ready(())
14821 })
14822 .await;
14823 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14824 drop(owner);
14825 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14826 .await
14827 .unwrap()
14828 .unwrap();
14829 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14830 assert!(execution.runner.state.resource_block.is_none());
14831 assert_eq!(
14832 execution.runner.state.liveness(false),
14833 crate::run::Liveness::Live
14834 );
14835 drop(execution);
14836 assert!(RunState::load(&id).unwrap().driver_exited);
14837 }
14838
14839 #[tokio::test]
14840 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14841 crate::run::pin_test_home();
14842 let mut runner = runner_at(RunStatus::Gating);
14843 runner.state.save().unwrap();
14844 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14847 assert!(runner.claim_driver().await.is_err());
14848 assert!(runner.state.driver_exited);
14849 assert_eq!(runner.state.driver_pid, None);
14850 let disk = RunState::load(&runner.state.id).unwrap();
14851 assert!(disk.driver_exited);
14852 assert_eq!(disk.driver_pid, None);
14853 assert_eq!(disk.status, RunStatus::Gating);
14854 }
14855
14856 #[tokio::test]
14857 async fn an_early_graph_error_releases_its_driver() {
14858 crate::run::pin_test_home();
14859 let mut runner = runner_at(RunStatus::Blocked);
14860 runner.state.config.graph.selected_implementer = Some("selected".into());
14861 runner.state.save().unwrap();
14862 assert!(runner.execute().await.is_err());
14863 let disk = RunState::load(&runner.state.id).unwrap();
14864 assert!(disk.driver_exited);
14865 assert_eq!(disk.driver_pid, None);
14866 assert_eq!(disk.status, RunStatus::Blocked);
14867 }
14868
14869 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14870 crate::run::MergeRepair {
14871 question: "repair-question".into(),
14872 head: "old-head".into(),
14873 pr: "https://github.com/o/r/pull/42".into(),
14874 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14875 gate: vec![outcome(Some(0), false)],
14876 gate_head: Some("old-head".into()),
14877 gate_fixes: Vec::new(),
14878 pre_gate: Vec::new(),
14879 phase,
14880 failure: None,
14881 }
14882 }
14883
14884 #[tokio::test]
14885 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14886 crate::run::pin_test_home();
14887 let mut runner = runner_at(RunStatus::Landing);
14888 runner.state.merge_repairs.push(merge_repair_request(
14889 crate::run::MergeRepairPhase::Requested,
14890 ));
14891 runner.state.save().unwrap();
14892 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14893 assert!(
14894 runner
14895 .execute()
14896 .await
14897 .unwrap_err()
14898 .to_string()
14899 .contains("already running")
14900 );
14901 assert_eq!(
14902 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14903 crate::run::MergeRepairPhase::Requested
14904 );
14905 drop(claim);
14906 let mut disk = runner.state.clone();
14907 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14908 disk.save().unwrap();
14909 assert!(
14910 runner
14911 .execute()
14912 .await
14913 .unwrap_err()
14914 .to_string()
14915 .contains("decision changed")
14916 );
14917 assert_eq!(
14918 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14919 crate::run::MergeRepairPhase::Complete
14920 );
14921 }
14922
14923 #[test]
14924 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14925 crate::run::pin_test_home();
14926 let mut runner = runner_at(RunStatus::Landing);
14927 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14928 runner.state.reviews = request.reviews.clone();
14929 runner.state.gate = request.gate.clone();
14930 runner.state.gate_ran = true;
14931 runner.state.land_approval = Some(crate::run::LandApproval {
14932 question: "old".into(),
14933 head: "old-head".into(),
14934 });
14935 runner.state.merge_repairs.push(request);
14936 runner.begin_merge_repair_review(0).unwrap();
14937 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14938 assert!(!runner.state.gate_status().ok());
14939 assert!(runner.state.land_approval.is_none());
14940 let disk = RunState::load(&runner.state.id).unwrap();
14941 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14942 assert!(disk.merge_repairs[0].gate[0].ok());
14943 assert_eq!(
14944 disk.merge_repairs[0].phase,
14945 crate::run::MergeRepairPhase::Reviewing
14946 );
14947 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14948 assert!(disk.reviews.is_empty());
14949 }
14950
14951 #[tokio::test]
14952 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14953 crate::run::pin_test_home();
14954 for phase in [
14955 crate::run::MergeRepairPhase::Fixing,
14956 crate::run::MergeRepairPhase::Requested,
14957 ] {
14958 let mut runner = runner_at(RunStatus::Landing);
14959 runner.state.config.graph.land_rounds = 0;
14960 runner.state.merge_repairs.push(merge_repair_request(phase));
14961 runner.state.save().unwrap();
14962 assert!(!runner.repair_merge_review().await.unwrap());
14963 assert_eq!(runner.state.status, RunStatus::Blocked);
14964 assert_eq!(
14965 runner.state.merge_repairs[0].phase,
14966 crate::run::MergeRepairPhase::Failed
14967 );
14968 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14969 assert!(
14970 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14971 "interrupted"
14972 } else {
14973 "budget exhausted"
14974 })
14975 );
14976 assert!(runner.state.merge.is_none());
14977 }
14978 }
14979
14980 #[test]
14984 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14985 crate::run::pin_test_home();
14986 let mut runner = runner_at(RunStatus::Implementing);
14987 let interrupt = Pause::new();
14988 runner.watch_interrupt(interrupt.clone());
14989
14990 interrupt.park_because("task a1b2 asked to run first");
14991
14992 assert!(runner.park_here().expect("park_here"));
14993 assert!(runner.state.parked);
14994 let last = runner.state.events.last().expect("a park event");
14995 assert_eq!(last.node, "park");
14996 assert!(
14997 last.message.contains("task a1b2 asked to run first"),
14998 "expected the interrupt reason in {:?}",
14999 last.message
15000 );
15001 }
15002
15003 #[test]
15006 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15007 crate::run::pin_test_home();
15008 let mut runner = runner_at(RunStatus::Implementing);
15009 runner.state.begin_pass();
15010 let shutdown = Pause::new();
15011 runner.on_pause(shutdown.clone());
15012 let upgrade = ParkCause::Upgrade {
15013 from: "0.121.0".to_owned(),
15014 to: Some("0.121.1".to_owned()),
15015 };
15016 shutdown.park_for(upgrade.clone());
15017 shutdown.park_for(ParkCause::Operator);
15018
15019 assert!(runner.park_here().expect("park_here"));
15020 assert_eq!(
15021 runner.state.passes[0].end,
15022 Some(crate::run::PassEnd::Parked { cause: upgrade })
15023 );
15024 let last = runner.state.events.last().expect("a park event");
15025 assert!(last.message.contains("0.121.0"), "{}", last.message);
15026 }
15027
15028 #[tokio::test]
15029 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15030 crate::run::pin_test_home();
15031 let mut runner = runner_at(RunStatus::Reviewing);
15032 runner.state.save().unwrap();
15033 let mut execution = runner.claim_driver().await.expect("first execution");
15034 assert_eq!(execution.runner.state.passes.len(), 1);
15035 execution
15036 .driver
15037 .finish(&mut execution.runner.state)
15038 .await
15039 .unwrap();
15040 }
15041
15042 #[tokio::test]
15043 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15044 crate::run::pin_test_home();
15045 let mut runner = runner_at(RunStatus::Reviewing);
15046 runner.state.config.agents = vec![conductor()];
15047 runner.state.config.graph.timeout_resource_wait = 0;
15048 runner.state.save().unwrap();
15049 let id = runner.state.id.clone();
15050 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15051 assert!(runner.claim_driver().await.is_err());
15052 drop(owner);
15053 drop(runner);
15054 let mut resumed = Runner::resume(&id).unwrap();
15055 assert_eq!(resumed.state.driver_claimed, Some(false));
15056 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15057 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15058 let mut execution = resumed
15059 .claim_driver()
15060 .await
15061 .expect("retry initial claim after reload");
15062 assert_eq!(execution.runner.state.driver_claimed, Some(true));
15063 assert_eq!(execution.runner.state.passes.len(), 1);
15064 execution
15065 .driver
15066 .finish(&mut execution.runner.state)
15067 .await
15068 .unwrap();
15069 }
15070
15071 #[tokio::test]
15072 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15073 crate::run::pin_test_home();
15074 let mut runner = runner_at(RunStatus::Reviewing);
15075 runner.state.config.agents = vec![conductor()];
15076 runner.state.save().unwrap();
15077 let id = runner.state.id.clone();
15078 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15079 let mut resumed = Runner::resume(&id).unwrap();
15080 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15081 assert!(resumed.claim_driver().await.is_err());
15082 assert!(RunState::load(&id).unwrap().passes.is_empty());
15083 drop(lease);
15084 let mut execution = resumed.claim_driver().await.unwrap();
15085 execution
15086 .driver
15087 .finish(&mut execution.runner.state)
15088 .await
15089 .unwrap();
15090 }
15091
15092 #[tokio::test]
15093 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15094 crate::run::pin_test_home();
15095 let mut runner = runner_at(RunStatus::Reviewing);
15096 let state = &mut runner.state;
15097 state.config.agents = vec![conductor()];
15098 state.driver_claimed = None;
15099 state.driver_pid = Some(std::process::id());
15100 state.driver_started_at = None;
15101 state.begin_pass();
15102 state.save().unwrap();
15103 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15104 std::fs::write(
15105 state.dir().join("driver.json"),
15106 serde_json::to_vec(&serde_json::json!({
15107 "pid": state.driver_pid, "started_at": null,
15108 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15109 }))
15110 .unwrap(),
15111 )
15112 .unwrap();
15113 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15114 drop(lease);
15115 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15116 let mut resumed = Runner::resume(&state.id).unwrap();
15117 assert!(!resumed.state.driver_exited);
15118 let mut execution = resumed
15119 .claim_driver()
15120 .await
15121 .expect("lease proves previous execution ended");
15122 assert_eq!(execution.runner.state.passes.len(), 2);
15123 execution
15124 .driver
15125 .finish(&mut execution.runner.state)
15126 .await
15127 .unwrap();
15128 }
15129
15130 #[tokio::test]
15131 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15132 crate::run::pin_test_home();
15133 let mut runner = runner_at(RunStatus::Reviewing);
15134 let mut json = serde_json::to_value(&runner.state).unwrap();
15135 json.as_object_mut().unwrap().remove("driver_claimed");
15136 runner.state = serde_json::from_value(json).unwrap();
15137 assert_eq!(runner.state.driver_claimed, None);
15138 runner.state.save().unwrap();
15139 let mut execution = runner
15140 .claim_driver()
15141 .await
15142 .expect("the execution lease, not a missing marker, excludes a second driver");
15143 assert_eq!(execution.runner.state.passes.len(), 1);
15144 execution
15145 .driver
15146 .finish(&mut execution.runner.state)
15147 .await
15148 .unwrap();
15149 }
15150
15151 #[tokio::test]
15152 async fn driver_admission_refuses_a_proven_live_driver() {
15153 crate::run::pin_test_home();
15154 let mut runner = runner_at(RunStatus::Reviewing);
15155 let pid = std::process::id();
15156 runner.state.driver_pid = Some(pid);
15157 runner.state.driver_started_at = crate::proc::process_started_at(pid);
15158 assert!(runner.state.driver_started_at.is_some());
15159 runner.state.save().unwrap();
15160 let error = runner
15161 .claim_driver()
15162 .await
15163 .err()
15164 .expect("a live driver must refuse");
15165 assert!(error.to_string().contains("live driver"), "{error:#}");
15166 let saved = RunState::load(&runner.state.id).unwrap();
15167 assert_eq!(saved.driver_pid, Some(pid));
15168 assert_eq!(saved.status, RunStatus::Reviewing);
15169 assert!(saved.passes.is_empty());
15170 }
15171
15172 #[test]
15173 fn parking_resource_halt_persists_the_pass_once() {
15174 crate::run::pin_test_home();
15175 let mut runner = runner_at(RunStatus::Reviewing);
15176 runner.state.begin_pass();
15177 runner.state.resource_block = Some(crate::run::ResourceBlock {
15178 node: "fix".into(),
15179 kind: "shared build cache".into(),
15180 detail: "cancelled".into(),
15181 at: Timestamp::now(),
15182 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15183 });
15184 let cause = ParkCause::Upgrade {
15185 from: "old".into(),
15186 to: Some("new".into()),
15187 };
15188 runner.pause.park_for(cause.clone());
15189 runner.stop_for_resource().unwrap();
15192 assert!(runner.resource_halt().unwrap());
15193 assert!(runner.park_here().unwrap());
15194 let saved = RunState::load(&runner.state.id).unwrap();
15195 assert!(saved.parked);
15196 assert_eq!(saved.status, RunStatus::Reviewing);
15197 assert!(saved.passes[0].ended_at.is_some());
15198 assert_eq!(
15199 saved.passes[0].end,
15200 Some(crate::run::PassEnd::Parked {
15201 cause: cause.clone()
15202 })
15203 );
15204 assert_eq!(
15205 saved.resource_block.unwrap().cause,
15206 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15207 );
15208 assert_eq!(
15209 saved
15210 .events
15211 .iter()
15212 .filter(|event| event.node == "park")
15213 .count(),
15214 1
15215 );
15216 }
15217
15218 #[test]
15226 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15227 crate::run::pin_test_home();
15228 let mut runner = runner_at(RunStatus::Implementing);
15229 let shutdown = Pause::new();
15230 runner.on_pause(shutdown.clone());
15231 let interrupt = Pause::new();
15232 runner.watch_interrupt(interrupt.clone());
15233
15234 assert!(!runner.park_here().expect("park_here"));
15236 assert!(!runner.state.parked);
15237
15238 interrupt.park_because("test");
15240 assert!(!shutdown.parked());
15241 assert!(runner.park_here().expect("park_here"));
15242 }
15243
15244 #[tokio::test]
15258 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15259 crate::run::pin_test_home();
15260 let mut runner = runner_at(RunStatus::Implementing);
15261 let interrupt = Pause::new();
15262 runner.watch_interrupt(interrupt.clone());
15263
15264 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15265 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15266
15267 let node = async move {
15271 started_tx.send(()).expect("send started");
15272 finish_rx.await.expect("recv finish");
15273 "node finished"
15274 };
15275
15276 let interrupter = async move {
15277 started_rx.await.expect("recv started");
15278 interrupt.park_because("higher-priority task waiting");
15280 tokio::task::yield_now().await;
15284 finish_tx.send(()).expect("send finish");
15285 };
15286
15287 let (node_result, ()) = tokio::join!(node, interrupter);
15288 assert_eq!(
15289 node_result, "node finished",
15290 "the in-flight call ran to completion"
15291 );
15292
15293 assert!(runner.park_here().expect("park_here"));
15296 assert!(runner.state.parked);
15297 }
15298
15299 #[test]
15305 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15306 crate::run::pin_test_home();
15307 let mut runner = runner_at(RunStatus::Judging);
15308 runner.state.config.agents = vec![conductor()];
15312 runner.state.candidates = vec![Candidate {
15313 index: 0,
15314 label: 'A',
15315 agent: "alpha".to_owned(),
15316 branch: "magi/x/A".to_owned(),
15317 worktree: PathBuf::from("/nonexistent/worktree"),
15318 summary: "did the thing".to_owned(),
15319 stat: "1 file changed".to_owned(),
15320 files: 1,
15321 commits: 1,
15322 empty: false,
15323 failed: None,
15324 verified_noop: None,
15325 duration_ms: 1234,
15326 folded: false,
15327 }];
15328 let run_id = runner.state.id.clone();
15329
15330 let interrupt = Pause::new();
15331 runner.watch_interrupt(interrupt.clone());
15332 interrupt.park_because("task c3d4 asked to run first");
15333 assert!(runner.park_here().expect("park_here"));
15334
15335 let resumed = Runner::resume(&run_id).expect("resume");
15336 assert_eq!(resumed.state.candidates.len(), 1);
15337 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15338 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15339 assert_eq!(resumed.state.status, runner.state.status);
15340 assert!(
15341 resumed.state.parked,
15342 "still parked until `execute` actually walks the graph again"
15343 );
15344 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15345 }
15346
15347 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15349 let mut q = ask::Question::new(
15350 run.to_owned(),
15351 "implement".to_owned(),
15352 "impl-A".to_owned(),
15353 "Which storage backend should the cache use?".to_owned(),
15354 String::new(),
15355 vec!["SQLite".to_owned(), "Redis".to_owned()],
15356 );
15357 store.put(&mut q).unwrap();
15358 q
15359 }
15360
15361 #[test]
15362 fn a_failed_runs_open_question_is_abandoned() {
15363 ask_test_home();
15364 let store = ask::Questions::open();
15365 let mut runner = runner_at(RunStatus::Failed);
15366 let run = runner.state.id.clone();
15367 let q = ask_open_question(&store, &run);
15368
15369 runner.settle_questions();
15370
15371 let back = store.get(&q.id).unwrap();
15372 assert!(
15373 !back.status.open(),
15374 "the seat that asked died with the run; nobody is left to read an answer"
15375 );
15376 assert!(
15377 back.detail.contains(&run) && back.detail.contains("failed"),
15378 "the reason names what the run became, not just that it is gone: {}",
15379 back.detail
15380 );
15381 }
15382
15383 #[test]
15384 fn a_merged_runs_open_question_is_abandoned_too() {
15385 ask_test_home();
15386 let store = ask::Questions::open();
15387 for status in [RunStatus::Merged, RunStatus::Ready] {
15390 let mut runner = runner_at(status);
15391 let run = runner.state.id.clone();
15392 let q = ask_open_question(&store, &run);
15393
15394 runner.settle_questions();
15395
15396 let back = store.get(&q.id).unwrap();
15397 assert!(
15398 !back.status.open(),
15399 "{status:?} run's question must not outlive the run"
15400 );
15401 }
15402 }
15403
15404 #[test]
15405 fn a_still_resumable_runs_open_question_is_left_alone() {
15406 ask_test_home();
15407 let store = ask::Questions::open();
15408 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15414 let mut runner = runner_at(status);
15415 let run = runner.state.id.clone();
15416 let q = ask_open_question(&store, &run);
15417
15418 runner.settle_questions();
15419
15420 let back = store.get(&q.id).unwrap();
15421 assert!(
15422 back.status.open(),
15423 "{status:?} is still alive; the question must still be waiting"
15424 );
15425 }
15426 }
15427
15428 #[test]
15429 fn settle_questions_never_touches_an_already_answered_question() {
15430 ask_test_home();
15431 let store = ask::Questions::open();
15432 let mut runner = runner_at(RunStatus::Failed);
15433 let run = runner.state.id.clone();
15434 let mut q = ask_open_question(&store, &run);
15435 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15436 .unwrap();
15437 store.put(&mut q).unwrap();
15438
15439 runner.settle_questions();
15444 runner.settle_questions();
15445
15446 let back = store.get(&q.id).unwrap();
15447 assert_eq!(
15448 back.status,
15449 ask::QuestionStatus::Answered,
15450 "a real answer is a decision on record, never overwritten by a sweep"
15451 );
15452 }
15453
15454 #[tokio::test]
15465 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15466 crate::run::pin_test_home();
15467 let tmp = tempfile::tempdir().expect("tempdir");
15468 let repo = tmp.path().join("repo");
15469 std::fs::create_dir_all(&repo).unwrap();
15470 init_repo(&repo);
15471
15472 let mut config = Config::default();
15473 config.graph.worktree_root = Some(tmp.path().join("wt"));
15474
15475 let mut state = RunState::new(
15476 repo.clone(),
15477 "main".to_owned(),
15478 "deadbeef".to_owned(),
15479 "task".to_owned(),
15480 config,
15481 );
15482 let root = state.worktree_root();
15483 let wt_a = root.join("cand-A");
15484 let wt_b = root.join("cand-B");
15485 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15486 .await
15487 .expect("worktree A");
15488 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15489 .await
15490 .expect("worktree B");
15491
15492 state.candidates = vec![
15493 Candidate {
15494 index: 0,
15495 label: 'A',
15496 agent: "alpha".to_owned(),
15497 branch: "magi/x/A".to_owned(),
15498 worktree: wt_a.clone(),
15499 summary: String::new(),
15500 stat: String::new(),
15501 files: 0,
15502 commits: 0,
15503 empty: false,
15504 failed: None,
15505 verified_noop: None,
15506 duration_ms: 0,
15507 folded: false,
15508 },
15509 Candidate {
15510 index: 1,
15511 label: 'B',
15512 agent: "beta".to_owned(),
15513 branch: "magi/x/B".to_owned(),
15514 worktree: wt_b.clone(),
15515 summary: String::new(),
15516 stat: String::new(),
15517 files: 0,
15518 commits: 0,
15519 empty: false,
15520 failed: None,
15521 verified_noop: None,
15522 duration_ms: 0,
15523 folded: false,
15524 },
15525 ];
15526 state.tally = Some(Tally {
15527 first_choice: BTreeMap::from([('A', 1)]),
15528 borda: BTreeMap::new(),
15529 winner: 'A',
15530 rankings: 1,
15531 unanimous_initial: true,
15532 deliberated: false,
15533 changed_votes: 0,
15534 unanimous_final: true,
15535 tie_break: None,
15536 judges: 1,
15537 present: 1,
15538 quorum: 1,
15539 met_quorum: true,
15540 uncontested: None,
15541 });
15542 state.status = RunStatus::Ready;
15543
15544 fold_run(&mut state, false, &crate::run::home())
15545 .await
15546 .expect("fold_run");
15547
15548 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15549 assert!(
15550 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15551 "the unmerged winner's branch survives"
15552 );
15553 assert!(
15554 !state.candidates[0].folded,
15555 "the winner is not marked folded"
15556 );
15557
15558 assert!(!wt_b.exists(), "the loser's worktree is removed");
15559 assert!(
15560 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15561 "the loser's branch is removed"
15562 );
15563 assert!(state.candidates[1].folded, "the loser is marked folded");
15564 }
15565
15566 #[tokio::test]
15567 #[cfg(unix)]
15568 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15569 use std::os::fd::AsRawFd as _;
15570 use std::time::Duration;
15571
15572 let tmp = tempfile::tempdir().unwrap();
15573 let repo = tmp.path().join("repo");
15574 std::fs::create_dir_all(&repo).unwrap();
15575 init_repo(&repo);
15576 let common = repo.join(".git");
15577 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15578 let home = tmp.path().join("home");
15579 let mut config = Config::default();
15580 config.coordination = crate::config::test_coordination();
15581 config.graph.worktree_root = Some(tmp.path().join("wt"));
15582 config.graph.timeout_resource_wait = 1;
15583 let mut state = RunState::new(
15584 repo.clone(),
15585 "main".into(),
15586 "deadbeef".into(),
15587 "task".into(),
15588 config,
15589 );
15590 let root = state.worktree_root();
15591 let candidate = root.join("cand-A");
15592 let reviewer = root.join("review-1");
15593 let branch = state.branch_for('A');
15594 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15595 .await
15596 .unwrap();
15597 git::worktree_add_detached(&repo, &reviewer, "main")
15598 .await
15599 .unwrap();
15600 git::acquire_worktree_config(&repo).await.unwrap();
15601 state.enabled_worktree_config = true;
15602 state.status = RunStatus::Ready;
15603 state.candidates.push(Candidate {
15604 index: 0,
15605 label: 'A',
15606 agent: "implement".into(),
15607 branch: branch.clone(),
15608 worktree: candidate.clone(),
15609 summary: String::new(),
15610 stat: String::new(),
15611 files: 0,
15612 commits: 0,
15613 empty: false,
15614 failed: None,
15615 verified_noop: None,
15616 duration_ms: 0,
15617 folded: false,
15618 });
15619 state.save_under(&home).unwrap();
15620 let config_before = std::fs::read(common.join("config")).unwrap();
15621 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15622 .await
15623 .unwrap();
15624 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15627 assert_eq!(
15628 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15629 0
15630 );
15631 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15632 assert!(
15633 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15634 .await
15635 .is_err()
15636 );
15637 assert!(candidate.exists());
15638 assert!(reviewer.exists());
15639 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15640 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15641 assert_eq!(
15642 git::git(&repo, &["worktree", "list", "--porcelain"])
15643 .await
15644 .unwrap(),
15645 worktrees_before
15646 );
15647 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15648 .await
15649 .unwrap()
15650 .unwrap_err();
15651 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15652 drop(fold);
15653 assert!(candidate.exists() && reviewer.exists());
15654 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15655 assert_eq!(
15656 state.resource_block.as_ref().unwrap().kind,
15657 "repository coordination"
15658 );
15659 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15660 drop(external);
15661 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15662 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15663 .await
15664 .unwrap()
15665 .unwrap();
15666 drop(fold);
15667 assert!(removed.contains(&branch));
15668 assert!(!root.exists());
15669 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15670 assert!(state.candidates[0].folded);
15671 assert!(!state.enabled_worktree_config);
15672 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15673 let enabled = git::git_raw(
15674 &repo,
15675 &["config", "--local", "--get", "extensions.worktreeConfig"],
15676 )
15677 .await
15678 .unwrap();
15679 assert!(!enabled.ok(), "the last config reference was released");
15680 }
15681
15682 #[tokio::test]
15685 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15686 let tmp = tempfile::tempdir().expect("tempdir");
15687 let repo = tmp.path().join("repo");
15688 std::fs::create_dir_all(&repo).unwrap();
15689 init_repo(&repo);
15690 let home = tmp.path().join("home");
15691
15692 let mut config = Config::default();
15693 config.graph.worktree_root = Some(tmp.path().join("wt"));
15694 let mut state = RunState::new(
15695 repo.clone(),
15696 "main".to_owned(),
15697 "deadbeef".to_owned(),
15698 "task".to_owned(),
15699 config,
15700 );
15701 git::git(&repo, &["branch", "magi/x/A", "main"])
15703 .await
15704 .expect("branch");
15705 state.candidates = vec![Candidate {
15706 index: 0,
15707 label: 'A',
15708 agent: "alpha".to_owned(),
15709 branch: "magi/x/A".to_owned(),
15710 worktree: state.worktree_root().join("cand-A"),
15711 summary: String::new(),
15712 stat: String::new(),
15713 files: 0,
15714 commits: 0,
15715 empty: false,
15716 failed: None,
15717 verified_noop: None,
15718 duration_ms: 0,
15719 folded: true,
15720 }];
15721 state.released_to = Some("20260901-000000-new1".to_owned());
15722 state.released_branches = vec!["magi/x/A".to_owned()];
15723
15724 fold_run(&mut state, true, &home).await.expect("fold_run");
15725 state.status = RunStatus::Merged;
15726 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15727 state.merge = Some(crate::run::MergeOutcome {
15728 mode: MergeMode::Pr,
15729 ok: true,
15730 detail: "confirmed merged".into(),
15731 empty: false,
15732 });
15733 fold_run(&mut state, true, &home)
15734 .await
15735 .expect("merged fold");
15736
15737 assert!(
15738 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15739 "the handed-over branch survives a fold"
15740 );
15741 }
15742
15743 #[tokio::test]
15747 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15748 let tmp = tempfile::tempdir().expect("tempdir");
15749 let repo = tmp.path().join("repo");
15750 std::fs::create_dir_all(&repo).unwrap();
15751 init_repo(&repo);
15752 let run = |args: &[&str]| {
15753 let out = std::process::Command::new("git")
15754 .quiet()
15755 .args(args)
15756 .current_dir(&repo)
15757 .output()
15758 .expect("spawn git");
15759 assert!(out.status.success(), "git {args:?}");
15760 };
15761 run(&["branch", "magi/x/A"]);
15762 run(&["checkout", "-q", "-b", "magi/x/B"]);
15763 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15764 run(&["add", "-A"]);
15765 run(&["commit", "-q", "-m", "work"]);
15766 run(&["checkout", "-q", "main"]);
15767
15768 let probe = RunState::new(
15772 repo.clone(),
15773 "main".to_owned(),
15774 "deadbeef".to_owned(),
15775 "task".to_owned(),
15776 Config::default(),
15777 );
15778 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15779 let bare = tmp.path().join("origin.git");
15780 let out = std::process::Command::new("git")
15781 .quiet()
15782 .args(["init", "-q", "--bare"])
15783 .arg(&bare)
15784 .output()
15785 .expect("spawn git");
15786 assert!(out.status.success(), "git init --bare");
15787 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15788 run(&["push", "-q", "origin", "main"]);
15789
15790 let mut state = RunState::new(
15791 repo.clone(),
15792 "main".to_owned(),
15793 "deadbeef".to_owned(),
15794 "task".to_owned(),
15795 Config::default(),
15796 );
15797 state.seeds = vec![refs::Seed {
15798 token: "magi/27b2/A".to_owned(),
15799 kind: refs::SeedKind::Unresolved,
15800 sha: String::new(),
15801 branch: true,
15802 detail: "no branch or commit named magi/27b2/A".to_owned(),
15803 }];
15804
15805 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15806 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15807 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15808 let detail = empty_candidate_detail(&state, "main");
15809 assert!(detail.starts_with("empty candidate"), "{detail}");
15810 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15811 assert!(detail.contains("--start-from"), "{detail}");
15812 }
15813
15814 #[tokio::test]
15823 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15824 let tmp = tempfile::tempdir().expect("tempdir");
15825 let repo = tmp.path().join("repo");
15826 std::fs::create_dir_all(&repo).unwrap();
15827 init_repo(&repo);
15828
15829 let mut config = Config::default();
15830 config.merge.mode = MergeMode::Local;
15831
15832 let mut state = RunState::new(
15833 repo.clone(),
15834 "main".to_owned(),
15835 "deadbeef".to_owned(),
15836 "task".to_owned(),
15837 config,
15838 );
15839 state.candidates = vec![Candidate {
15840 index: 0,
15841 label: 'A',
15842 agent: "alpha".to_owned(),
15843 branch: "does-not-exist".to_owned(),
15844 worktree: repo.clone(),
15845 summary: String::new(),
15846 stat: String::new(),
15847 files: 0,
15848 commits: 0,
15849 empty: false,
15850 failed: None,
15851 verified_noop: None,
15852 duration_ms: 0,
15853 folded: false,
15854 }];
15855 state.tally = Some(Tally {
15856 first_choice: BTreeMap::from([('A', 1)]),
15857 borda: BTreeMap::new(),
15858 winner: 'A',
15859 rankings: 1,
15860 unanimous_initial: true,
15861 deliberated: false,
15862 changed_votes: 0,
15863 unanimous_final: true,
15864 tie_break: None,
15865 judges: 0,
15866 present: 0,
15867 quorum: 0,
15868 met_quorum: true,
15869 uncontested: Some("only candidate A produced a change".to_owned()),
15870 });
15871 state.reviews = vec![ReviewRound {
15872 round: 1,
15873 head: "deadbeef".to_owned(),
15874 verified_head: None,
15875 verified_at: None,
15876 reviews: Vec::new(),
15877 e2e: Vec::new(),
15878 fix: None,
15879 blocking: 0,
15880 answered: 0,
15881 expected: 0,
15882 clean: true,
15883 verify_retried: false,
15884 e2e_deferred: false,
15885 e2e_defer_reason: None,
15886 progressed: false,
15887 vote_split: false,
15888 reconsideration: Vec::new(),
15889 verdict: None,
15890 }];
15891 state.gate = vec![CommandOutcome {
15892 command: "test".to_owned(),
15893 code: Some(0),
15894 output_tail: String::new(),
15895 duration_ms: 0,
15896 resource_blocked: false,
15897 }];
15898 state.gate_ran = true;
15899 state.status = RunStatus::Ready;
15904 state.merge = Some(MergeOutcome {
15905 mode: MergeMode::Local,
15906 ok: false,
15907 detail: "already concluded".to_owned(),
15908 empty: false,
15909 });
15910
15911 let mut runner = Runner {
15912 state,
15913 roles: ResolvedRoles {
15914 implementers: Vec::new(),
15915 judges: Vec::new(),
15916 reviewers: Vec::new(),
15917 fixer: None,
15918 conductor: conductor(),
15919 implementer_roster: Vec::new(),
15920 judge_roster: Vec::new(),
15921 reviewer_roster: Vec::new(),
15922 },
15923 sem: Arc::new(Semaphore::new(1)),
15924 pause: Pause::new(),
15925 interrupt: Pause::new(),
15926 };
15927
15928 runner.merge().await.expect("merge");
15929
15930 assert_eq!(
15931 runner.state.status,
15932 RunStatus::Ready,
15933 "a concluded run's status must not change on reentry"
15934 );
15935 assert_eq!(
15936 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15937 Some("already concluded"),
15938 "merge must not run again once the node already recorded an outcome"
15939 );
15940 }
15941
15942 #[tokio::test]
15951 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15952 let tmp = tempfile::tempdir().expect("tempdir");
15953 let repo = tmp.path().join("repo");
15954 std::fs::create_dir_all(&repo).unwrap();
15955 init_repo(&repo);
15956
15957 let mut config = Config::default();
15958 config.merge.mode = MergeMode::Local;
15959
15960 let mut state = RunState::new(
15961 repo.clone(),
15962 "main".to_owned(),
15963 "deadbeef".to_owned(),
15964 "task".to_owned(),
15965 config,
15966 );
15967 state.candidates = vec![Candidate {
15968 index: 0,
15969 label: 'A',
15970 agent: "alpha".to_owned(),
15971 branch: "does-not-exist".to_owned(),
15972 worktree: repo.clone(),
15973 summary: String::new(),
15974 stat: String::new(),
15975 files: 0,
15976 commits: 0,
15977 empty: false,
15978 failed: None,
15979 verified_noop: None,
15980 duration_ms: 0,
15981 folded: false,
15982 }];
15983 state.tally = Some(Tally {
15984 first_choice: BTreeMap::from([('A', 1)]),
15985 borda: BTreeMap::new(),
15986 winner: 'A',
15987 rankings: 1,
15988 unanimous_initial: true,
15989 deliberated: false,
15990 changed_votes: 0,
15991 unanimous_final: true,
15992 tie_break: None,
15993 judges: 0,
15994 present: 0,
15995 quorum: 0,
15996 met_quorum: true,
15997 uncontested: Some("only candidate A produced a change".to_owned()),
15998 });
15999 state.reviews = vec![ReviewRound {
16000 round: 1,
16001 head: "deadbeef".to_owned(),
16002 verified_head: None,
16003 verified_at: None,
16004 reviews: Vec::new(),
16005 e2e: Vec::new(),
16006 fix: None,
16007 blocking: 0,
16008 answered: 0,
16009 expected: 0,
16010 clean: true,
16011 verify_retried: false,
16012 e2e_deferred: false,
16013 e2e_defer_reason: None,
16014 progressed: false,
16015 vote_split: false,
16016 reconsideration: Vec::new(),
16017 verdict: None,
16018 }];
16019 state.gate = Vec::new();
16021 state.gate_ran = false;
16022 state.status = RunStatus::Gating;
16023
16024 let mut runner = Runner {
16025 state,
16026 roles: ResolvedRoles {
16027 implementers: Vec::new(),
16028 judges: Vec::new(),
16029 reviewers: Vec::new(),
16030 fixer: None,
16031 conductor: conductor(),
16032 implementer_roster: Vec::new(),
16033 judge_roster: Vec::new(),
16034 reviewer_roster: Vec::new(),
16035 },
16036 sem: Arc::new(Semaphore::new(1)),
16037 pause: Pause::new(),
16038 interrupt: Pause::new(),
16039 };
16040
16041 runner.merge().await.expect("merge");
16042
16043 assert!(
16044 runner.state.merge.is_none(),
16045 "an empty gate must never be read as a passing one: {:?}",
16046 runner.state.merge
16047 );
16048 }
16049
16050 #[tokio::test]
16057 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16058 let tmp = tempfile::tempdir().expect("tempdir");
16059 let repo = tmp.path().join("repo");
16060 std::fs::create_dir_all(&repo).unwrap();
16061 init_repo(&repo);
16062
16063 let config = Config::default();
16065
16066 let mut state = RunState::new(
16067 repo.clone(),
16068 "main".to_owned(),
16069 "deadbeef".to_owned(),
16070 "task".to_owned(),
16071 config,
16072 );
16073 state.candidates = vec![Candidate {
16074 index: 0,
16075 label: 'A',
16076 agent: "alpha".to_owned(),
16077 branch: "does-not-exist".to_owned(),
16078 worktree: repo.clone(),
16079 summary: String::new(),
16080 stat: String::new(),
16081 files: 0,
16082 commits: 0,
16083 empty: false,
16084 failed: None,
16085 verified_noop: None,
16086 duration_ms: 0,
16087 folded: false,
16088 }];
16089 state.tally = Some(Tally {
16090 first_choice: BTreeMap::from([('A', 1)]),
16091 borda: BTreeMap::new(),
16092 winner: 'A',
16093 rankings: 1,
16094 unanimous_initial: true,
16095 deliberated: false,
16096 changed_votes: 0,
16097 unanimous_final: true,
16098 tie_break: None,
16099 judges: 0,
16100 present: 0,
16101 quorum: 0,
16102 met_quorum: true,
16103 uncontested: Some("only candidate A produced a change".to_owned()),
16104 });
16105 state.reviews = vec![ReviewRound {
16106 round: 1,
16107 head: "deadbeef".to_owned(),
16108 verified_head: None,
16109 verified_at: None,
16110 reviews: Vec::new(),
16111 e2e: Vec::new(),
16112 fix: None,
16113 blocking: 0,
16114 answered: 0,
16115 expected: 0,
16116 clean: true,
16117 verify_retried: false,
16118 e2e_deferred: false,
16119 e2e_defer_reason: None,
16120 progressed: false,
16121 vote_split: false,
16122 reconsideration: Vec::new(),
16123 verdict: None,
16124 }];
16125
16126 let mut runner = Runner {
16127 state,
16128 roles: ResolvedRoles {
16129 implementers: Vec::new(),
16130 judges: Vec::new(),
16131 reviewers: Vec::new(),
16132 fixer: None,
16133 conductor: conductor(),
16134 implementer_roster: Vec::new(),
16135 judge_roster: Vec::new(),
16136 reviewer_roster: Vec::new(),
16137 },
16138 sem: Arc::new(Semaphore::new(1)),
16139 pause: Pause::new(),
16140 interrupt: Pause::new(),
16141 };
16142
16143 runner.gate().await.expect("gate");
16144 assert!(
16145 runner.state.gate_ran,
16146 "zero configured commands is still a real attempt, not an unrun gate"
16147 );
16148 assert!(runner.state.gate.is_empty());
16149 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16150 assert_ne!(
16151 runner.state.status,
16152 RunStatus::Blocked,
16153 "a gate with nothing to check must not read as failed"
16154 );
16155
16156 runner.merge().await.expect("merge");
16157 assert_eq!(
16158 runner.state.status,
16159 RunStatus::Ready,
16160 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16161 );
16162 }
16163
16164 #[tokio::test]
16175 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16176 crate::run::pin_test_home();
16177 let home = crate::run::home();
16178
16179 let tmp = tempfile::tempdir().expect("tempdir");
16180 let repo = tmp.path().join("repo");
16181 std::fs::create_dir_all(&repo).unwrap();
16182 init_repo(&repo);
16183 let cache_dir = tmp.path().join("target");
16186
16187 let mut config = Config::default();
16188 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16189 config.graph.timeout_verify = Some(2);
16192 config.graph.timeout_resource_wait = 2;
16193
16194 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16195 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16196 .expect("no io error acquiring directly")
16197 {
16198 crate::cache::AcquireOutcome::Acquired(g) => g,
16199 crate::cache::AcquireOutcome::Busy(b) => {
16200 panic!("expected the direct acquire to win the lease first: {b:?}")
16201 }
16202 };
16203
16204 let mut state = RunState::new(
16205 repo.clone(),
16206 "main".to_owned(),
16207 "deadbeef".to_owned(),
16208 "task".to_owned(),
16209 config,
16210 );
16211 state.candidates = vec![Candidate {
16212 index: 0,
16213 label: 'A',
16214 agent: "alpha".to_owned(),
16215 branch: "does-not-exist".to_owned(),
16216 worktree: repo.clone(),
16217 summary: String::new(),
16218 stat: String::new(),
16219 files: 0,
16220 commits: 0,
16221 empty: false,
16222 failed: None,
16223 verified_noop: None,
16224 duration_ms: 0,
16225 folded: false,
16226 }];
16227 state.tally = Some(Tally {
16228 first_choice: BTreeMap::from([('A', 1)]),
16229 borda: BTreeMap::new(),
16230 winner: 'A',
16231 rankings: 1,
16232 unanimous_initial: true,
16233 deliberated: false,
16234 changed_votes: 0,
16235 unanimous_final: true,
16236 tie_break: None,
16237 judges: 0,
16238 present: 0,
16239 quorum: 0,
16240 met_quorum: true,
16241 uncontested: Some("only candidate A produced a change".to_owned()),
16242 });
16243 state.reviews = vec![ReviewRound {
16244 round: 1,
16245 head: "deadbeef".to_owned(),
16246 verified_head: None,
16247 verified_at: None,
16248 reviews: Vec::new(),
16249 e2e: Vec::new(),
16250 fix: None,
16251 blocking: 0,
16252 answered: 0,
16253 expected: 0,
16254 clean: true,
16255 verify_retried: false,
16256 e2e_deferred: false,
16257 e2e_defer_reason: None,
16258 progressed: false,
16259 vote_split: false,
16260 reconsideration: Vec::new(),
16261 verdict: None,
16262 }];
16263
16264 let mut runner = Runner {
16265 state,
16266 roles: ResolvedRoles {
16267 implementers: Vec::new(),
16268 judges: Vec::new(),
16269 reviewers: Vec::new(),
16270 fixer: None,
16271 conductor: conductor(),
16272 implementer_roster: Vec::new(),
16273 judge_roster: Vec::new(),
16274 reviewer_roster: Vec::new(),
16275 },
16276 sem: Arc::new(Semaphore::new(1)),
16277 pause: Pause::new(),
16278 interrupt: Pause::new(),
16279 };
16280
16281 let started = std::time::Instant::now();
16282 runner.gate().await.expect("gate");
16283 assert!(
16284 started.elapsed() < Duration::from_secs(1),
16285 "a gate with nothing to run must never wait on a lease it never needed"
16286 );
16287 assert!(
16288 runner.state.gate_ran,
16289 "zero commands is still a real, immediate attempt"
16290 );
16291 assert!(runner.state.gate.is_empty());
16292 assert_ne!(
16293 runner.state.status,
16294 RunStatus::Blocked,
16295 "must not read as resource-blocked on a lease it never asked for"
16296 );
16297 }
16298
16299 #[cfg(unix)]
16300 #[tokio::test]
16301 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16302 crate::run::pin_test_home();
16303 let tmp = tempfile::tempdir().unwrap();
16304 let mut config = Config::default();
16305 config.disk.min_free_bytes = 0;
16306 let mut state = RunState::new(
16307 tmp.path().to_owned(),
16308 "main".into(),
16309 "base".into(),
16310 "verification evidence".into(),
16311 config,
16312 );
16313 let (outcomes, pids) = run_commands(
16314 &mut state,
16315 "gate",
16316 "gate",
16317 1,
16318 &["sh".into(), "-c".into()],
16319 &[
16320 "printf before-timeout; exec sleep 30".into(),
16321 "kill -TERM $$".into(),
16322 "exit 7".into(),
16323 "printf success".into(),
16324 ],
16325 tmp.path(),
16326 Duration::from_secs(1),
16327 )
16328 .await;
16329 wait_for_timed_out_children_to_die(&pids).await;
16330 assert_eq!(
16331 pids.len(),
16332 1,
16333 "unleased timeout retains its death-check PID"
16334 );
16335 assert_eq!(outcomes.len(), 4);
16336 assert_eq!(outcomes[0].code, None);
16337 assert!(outcomes[0].output_tail.contains("before-timeout"));
16338 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16339 assert_eq!(outcomes[1].code, None);
16340 assert!(outcomes[1].output_tail.contains("command terminated:"));
16341 assert!(!outcomes[1].output_tail.contains("timed out"));
16342 assert_eq!(outcomes[2].code, Some(7));
16343 assert!(!outcomes[2].output_tail.contains("timed out"));
16344 assert_eq!(outcomes[3].code, Some(0));
16345 assert_eq!(outcomes[3].output_tail, "success");
16346 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16347 }
16348
16349 #[tokio::test]
16359 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16360 crate::run::pin_test_home();
16361
16362 let tmp = tempfile::tempdir().expect("tempdir");
16363 let repo = tmp.path().join("repo");
16364 std::fs::create_dir_all(&repo).unwrap();
16365 init_repo(&repo);
16366
16367 let mut config = Config::default();
16368 config.verify.gate = vec![
16369 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16370 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16371 .to_owned(),
16372 ];
16373
16374 let mut state = RunState::new(
16375 repo.clone(),
16376 "main".to_owned(),
16377 "deadbeef".to_owned(),
16378 "task".to_owned(),
16379 config,
16380 );
16381 let run_id = state.id.clone();
16382 state.candidates = vec![Candidate {
16383 index: 0,
16384 label: 'A',
16385 agent: "alpha".to_owned(),
16386 branch: "does-not-exist".to_owned(),
16387 worktree: repo.clone(),
16388 summary: String::new(),
16389 stat: String::new(),
16390 files: 0,
16391 commits: 0,
16392 empty: false,
16393 failed: None,
16394 verified_noop: None,
16395 duration_ms: 0,
16396 folded: false,
16397 }];
16398 state.tally = Some(Tally {
16399 first_choice: BTreeMap::from([('A', 1)]),
16400 borda: BTreeMap::new(),
16401 winner: 'A',
16402 rankings: 1,
16403 unanimous_initial: true,
16404 deliberated: false,
16405 changed_votes: 0,
16406 unanimous_final: true,
16407 tie_break: None,
16408 judges: 0,
16409 present: 0,
16410 quorum: 0,
16411 met_quorum: true,
16412 uncontested: Some("only candidate A produced a change".to_owned()),
16413 });
16414 state.reviews = vec![ReviewRound {
16415 round: 1,
16416 head: "deadbeef".to_owned(),
16417 verified_head: None,
16418 verified_at: None,
16419 reviews: Vec::new(),
16420 e2e: Vec::new(),
16421 fix: None,
16422 blocking: 0,
16423 answered: 0,
16424 expected: 0,
16425 clean: true,
16426 verify_retried: false,
16427 e2e_deferred: false,
16428 e2e_defer_reason: None,
16429 progressed: false,
16430 vote_split: false,
16431 reconsideration: Vec::new(),
16432 verdict: None,
16433 }];
16434
16435 let mut runner = Runner {
16436 state,
16437 roles: ResolvedRoles {
16438 implementers: Vec::new(),
16439 judges: Vec::new(),
16440 reviewers: Vec::new(),
16441 fixer: None,
16442 conductor: conductor(),
16443 implementer_roster: Vec::new(),
16444 judge_roster: Vec::new(),
16445 reviewer_roster: Vec::new(),
16446 },
16447 sem: Arc::new(Semaphore::new(1)),
16448 pause: Pause::new(),
16449 interrupt: Pause::new(),
16450 };
16451
16452 let started_marker = repo.join("started.marker");
16453 let release_marker = repo.join("release.marker");
16454 let poller = tokio::spawn(async move {
16455 for _ in 0..100 {
16460 if started_marker.exists()
16461 && let Ok(s) = crate::run::RunState::load(&run_id)
16462 && let Some(a) = s.active.get("gate")
16463 {
16464 std::fs::write(&release_marker, b"go").expect("release marker");
16465 return Some(a.clone());
16466 }
16467 tokio::time::sleep(Duration::from_millis(50)).await;
16468 }
16469 None
16470 });
16471
16472 runner.gate().await.expect("gate");
16473 let captured = poller.await.expect("poller task");
16474 let captured = captured.expect(
16475 "the poller never saw a `gate` task entry in run.json while the command was \
16476 still blocked on its own release marker",
16477 );
16478
16479 assert_eq!(captured.task.as_deref(), Some("gate"));
16480 assert_eq!(captured.node, "gate");
16481 assert_eq!(captured.index, Some(1));
16482 assert_eq!(captured.total, Some(1));
16483 assert!(
16484 captured
16485 .command
16486 .as_deref()
16487 .is_some_and(|c| c.contains("started.marker")),
16488 "{captured:?}"
16489 );
16490
16491 assert!(
16492 runner.state.active.is_empty(),
16493 "the entry must be cleared once the command actually finished: {:?}",
16494 runner.state.active
16495 );
16496 assert!(runner.state.gate_ran);
16497 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16498 }
16499
16500 #[tokio::test]
16503 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16504 use crate::verdict::{Finding, ReviewVote, Severity};
16505 crate::run::pin_test_home();
16506 let tmp = tempfile::tempdir().expect("tempdir");
16507 let repo = tmp.path().join("repo");
16508 std::fs::create_dir_all(&repo).unwrap();
16509 init_repo(&repo);
16510
16511 for (severity, vote, expect) in [
16512 (Severity::Major, ReviewVote::Reject, true),
16513 (Severity::Minor, ReviewVote::Reject, false),
16514 (Severity::Major, ReviewVote::Approve, false),
16515 ] {
16516 let mut round = review_round(false, 1, 1, 1, false, true);
16517 round.reviews = vec![ReviewRecord {
16518 reviewer: 1,
16519 agent: "alpha".to_owned(),
16520 summary: String::new(),
16521 findings: vec![Finding {
16522 id: "R1-1-1".to_owned(),
16523 severity,
16524 file: None,
16525 line: None,
16526 title: "t".to_owned(),
16527 detail: String::new(),
16528 }],
16529 vote: Some(vote),
16530 failed: None,
16531 duration_ms: 0,
16532 attempts: 0,
16533 }];
16534 let mut state = RunState::new(
16535 repo.clone(),
16536 "main".to_owned(),
16537 "deadbeef".to_owned(),
16538 "task".to_owned(),
16539 Config::default(),
16540 );
16541 state.reviews = vec![round];
16542 let mut runner = Runner {
16543 state,
16544 roles: ResolvedRoles {
16545 implementers: Vec::new(),
16546 judges: Vec::new(),
16547 reviewers: Vec::new(),
16548 fixer: None,
16549 conductor: conductor(),
16550 implementer_roster: Vec::new(),
16551 judge_roster: Vec::new(),
16552 reviewer_roster: Vec::new(),
16553 },
16554 sem: Arc::new(Semaphore::new(1)),
16555 pause: Pause::new(),
16556 interrupt: Pause::new(),
16557 };
16558 let shell = runner.state.config.shell();
16559 runner
16560 .stop_reviewing("round budget spent", &shell, &repo)
16561 .await
16562 .expect("stop_reviewing");
16563 assert_eq!(runner.state.status, RunStatus::Gating);
16564 assert_eq!(
16565 runner.state.contested_handoff.is_some(),
16566 expect,
16567 "{severity:?} + {vote:?}"
16568 );
16569 }
16570 }
16571
16572 #[tokio::test]
16585 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16586 crate::run::pin_test_home();
16587 let home = crate::run::home();
16588
16589 let tmp = tempfile::tempdir().expect("tempdir");
16590 let repo = tmp.path().join("repo");
16591 std::fs::create_dir_all(&repo).unwrap();
16592 init_repo(&repo);
16593 let head = crate::git::rev_parse(&repo, "HEAD")
16594 .await
16595 .expect("rev-parse");
16596 let cache_dir = tmp.path().join("target");
16599
16600 let mut config = Config::default();
16601 config.verify.e2e = vec![format!(
16602 "CARGO_TARGET_DIR='{}' test -f README.md",
16603 cache_dir.display()
16604 )];
16605 config.graph.review_rounds = 1;
16606 config.graph.timeout_verify = Some(2);
16609 config.graph.timeout_resource_wait = 2;
16610
16611 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16612 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16613 .expect("no io error acquiring directly")
16614 {
16615 crate::cache::AcquireOutcome::Acquired(g) => g,
16616 crate::cache::AcquireOutcome::Busy(b) => {
16617 panic!("expected the direct acquire to win the lease first: {b:?}")
16618 }
16619 };
16620
16621 let mut state = RunState::new(
16622 repo.clone(),
16623 "main".to_owned(),
16624 head.clone(),
16625 "task".to_owned(),
16626 config,
16627 );
16628 state.candidates = vec![Candidate {
16629 index: 0,
16630 label: 'A',
16631 agent: "alpha".to_owned(),
16632 branch: "does-not-exist".to_owned(),
16633 worktree: repo.clone(),
16634 summary: String::new(),
16635 stat: String::new(),
16636 files: 0,
16637 commits: 0,
16638 empty: false,
16639 failed: None,
16640 verified_noop: None,
16641 duration_ms: 0,
16642 folded: false,
16643 }];
16644 state.tally = Some(Tally {
16645 first_choice: BTreeMap::from([('A', 1)]),
16646 borda: BTreeMap::new(),
16647 winner: 'A',
16648 rankings: 1,
16649 unanimous_initial: true,
16650 deliberated: false,
16651 changed_votes: 0,
16652 unanimous_final: true,
16653 tie_break: None,
16654 judges: 0,
16655 present: 0,
16656 quorum: 0,
16657 met_quorum: true,
16658 uncontested: Some("only candidate A produced a change".to_owned()),
16659 });
16660 state.reviews = vec![ReviewRound {
16664 round: 1,
16665 head: head.clone(),
16666 verified_head: None,
16667 verified_at: None,
16668 reviews: Vec::new(),
16669 e2e: Vec::new(),
16670 fix: None,
16671 blocking: 1,
16672 answered: 1,
16673 expected: 1,
16674 clean: false,
16675 verify_retried: false,
16676 e2e_deferred: true,
16677 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16678 progressed: false,
16679 vote_split: false,
16680 reconsideration: Vec::new(),
16681 verdict: None,
16682 }];
16683
16684 let mut runner = Runner {
16685 state,
16686 roles: ResolvedRoles {
16687 implementers: Vec::new(),
16688 judges: Vec::new(),
16689 reviewers: Vec::new(),
16690 fixer: None,
16691 conductor: conductor(),
16692 implementer_roster: Vec::new(),
16693 judge_roster: Vec::new(),
16694 reviewer_roster: Vec::new(),
16695 },
16696 sem: Arc::new(Semaphore::new(1)),
16697 pause: Pause::new(),
16698 interrupt: Pause::new(),
16699 };
16700
16701 let shell = runner.state.config.shell();
16702 runner
16703 .stop_reviewing("round budget spent", &shell, &repo)
16704 .await
16705 .expect("stop_reviewing");
16706
16707 let last = runner.state.reviews.last().expect("round record");
16708 assert_eq!(
16709 last.e2e_status(),
16710 E2eStatus::ResourceBlocked,
16711 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16712 failed: {last:?}"
16713 );
16714 assert_eq!(
16715 last.verified_head.as_deref(),
16716 Some(head.as_str()),
16717 "which commit this attempt targeted is known even though nothing finished checking \
16718 it"
16719 );
16720 let first_attempt_at = last
16721 .verified_at
16722 .expect("when this attempt ran is known too");
16723 assert_ne!(
16724 runner.state.status,
16725 RunStatus::Blocked,
16726 "contention is evidence about the machine, not the patch — it must not settle the \
16727 run as blocked: {:?}",
16728 runner.state.status
16729 );
16730 assert!(
16731 !runner
16732 .state
16733 .events
16734 .iter()
16735 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16736 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16737 runner.state.events
16738 );
16739
16740 runner
16745 .stop_reviewing("round budget spent", &shell, &repo)
16746 .await
16747 .expect("stop_reviewing retry");
16748 assert_eq!(
16749 runner.state.reviews.len(),
16750 1,
16751 "no new round was started: {:?}",
16752 runner.state.reviews
16753 );
16754 let last = runner.state.reviews.last().expect("round record");
16755 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16756 assert!(
16757 last.verified_at.expect("still known") > first_attempt_at,
16758 "a second reentry must be a fresh attempt, not a stale copy of the first"
16759 );
16760 assert_ne!(runner.state.status, RunStatus::Blocked);
16761
16762 held.release();
16763 }
16764
16765 #[cfg(unix)]
16766 #[tokio::test]
16767 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
16768 ask_test_home();
16769 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
16770 let tmp = tempfile::tempdir().unwrap();
16771 let repo = tmp.path().join("repo");
16772 std::fs::create_dir(&repo).unwrap();
16773 init_repo(&repo);
16774 let run_git = |args: &[&str]| {
16775 let out = std::process::Command::new("git")
16776 .args(args)
16777 .current_dir(&repo)
16778 .output()
16779 .unwrap();
16780 assert!(
16781 out.status.success(),
16782 "{args:?}: {}",
16783 String::from_utf8_lossy(&out.stderr)
16784 );
16785 };
16786 let origin = tmp.path().join("origin.git");
16787 run_git(&["init", "--bare", origin.to_str().unwrap()]);
16788 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
16789 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
16790 run_git(&["add", "spec.rs"]);
16791 run_git(&["commit", "-m", "initial spec"]);
16792 run_git(&["push", "origin", "main"]);
16793 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16794 run_git(&["checkout", "-b", "fix/review"]);
16795 std::fs::write(
16796 repo.join("check.rs"),
16797 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
16798 )
16799 .unwrap();
16800 run_git(&["add", "check.rs"]);
16801 run_git(&["commit", "-m", "use spec"]);
16802 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
16803 run_git(&["checkout", "main"]);
16804 std::fs::write(
16805 repo.join("spec.rs"),
16806 "struct AgentSpec { max_turn_secs: u64 }\n",
16807 )
16808 .unwrap();
16809 run_git(&["add", "spec.rs"]);
16810 run_git(&["commit", "-m", "extend spec"]);
16811 run_git(&["push", "origin", "main"]);
16812 run_git(&["checkout", "fix/review"]);
16813
16814 let mut runner = runner_at(RunStatus::Reviewing);
16815 runner.state.repo = repo.clone();
16816 runner.state.base_commit = base;
16817 runner.state.pr_head = Some("fix/review".into());
16818 runner.state.config.graph.review_rounds = max_rounds;
16819 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16820 runner.state.config.graph.sessions = false;
16821 runner.state.config.disk.min_free_bytes = 0;
16822 runner.state.config.verify.e2e =
16823 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
16824 runner.state.candidates = vec![Candidate {
16825 index: 0,
16826 label: 'A',
16827 agent: "fixture".into(),
16828 branch: "fix/review".into(),
16829 worktree: repo.clone(),
16830 summary: String::new(),
16831 stat: String::new(),
16832 files: 1,
16833 commits: 1,
16834 empty: false,
16835 failed: None,
16836 verified_noop: None,
16837 duration_ms: 0,
16838 folded: false,
16839 }];
16840 runner.state.tally = Some(Tally {
16841 first_choice: BTreeMap::new(),
16842 borda: BTreeMap::new(),
16843 winner: 'A',
16844 rankings: 0,
16845 unanimous_initial: true,
16846 deliberated: false,
16847 changed_votes: 0,
16848 unanimous_final: true,
16849 tie_break: None,
16850 judges: 0,
16851 present: 0,
16852 quorum: 0,
16853 met_quorum: true,
16854 uncontested: Some("review-only".into()),
16855 });
16856 let mut last = review_round(false, 0, 1, 1, false, false);
16857 last.head = reviewed.clone();
16858 last.e2e_deferred = true;
16859 last.e2e[0].resource_blocked = true;
16860 last.e2e[0].output_tail = "waiting for the shared build cache".into();
16861 if max_rounds == 1 {
16862 last.e2e.clear();
16863 }
16864 runner.state.reviews = vec![last];
16865 let calls = tmp.path().join("fix-calls");
16866 let mut fixer = spec("repair");
16867 fixer
16868 .env
16869 .insert("FIX_CALLS".into(), calls.display().to_string());
16870 fixer.command = vec!["sh".into(), "-c".into(), r#"
16871 echo fix >> "$FIX_CALLS"
16872 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
16873 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
16874 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
16875 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
16876 "#.into()];
16877 let mut reviewer = spec("reviewer");
16878 reviewer.command = vec![
16879 "sh".into(),
16880 "-c".into(),
16881 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
16882 .into(),
16883 ];
16884 runner.roles.fixer = Some(vec![fixer]);
16885 runner.roles.reviewers = vec![reviewer];
16886
16887 runner.sync_to_base().await.unwrap();
16888 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
16889 assert_ne!(rebased, reviewed);
16890 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
16891 if reentry {
16892 let shell = runner.state.config.shell();
16895 runner
16896 .stop_unresolved_review("deferred verification", &shell, &repo)
16897 .await
16898 .unwrap();
16899 assert!(runner.catchup_repairable());
16900 assert!(!calls.exists());
16901 runner.state = RunState::load(&runner.state.id).unwrap();
16902 assert!(runner.catchup_repairable());
16903 }
16904 runner.review_loop().await.unwrap();
16905 let first = &runner.state.reviews[0];
16906 assert_eq!(first.head, reviewed);
16907 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
16908 assert!(
16909 first.verify_retried,
16910 "a reproduced build failure gets exactly one retry: {:?}",
16911 first.e2e
16912 );
16913 assert!(first.e2e[0].output_tail.contains("E0063"));
16914 assert!(!runner.catchup_repairable());
16915 if max_rounds == 2 {
16916 assert_eq!(
16917 runner.state.status,
16918 RunStatus::Gating,
16919 "{}",
16920 crate::report::run(&runner.state)
16921 );
16922 assert_eq!(runner.state.reviews.len(), 2);
16923 assert!(first.fix.as_ref().unwrap().committed);
16924 assert!(runner.state.reviews[1].clean);
16925 assert_eq!(
16926 runner
16927 .state
16928 .reviews
16929 .iter()
16930 .filter(|r| r.fix.is_some())
16931 .count(),
16932 1
16933 );
16934 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
16935 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
16936 } else {
16937 assert_eq!(runner.state.status, RunStatus::Blocked);
16938 assert!(first.fix.is_none());
16939 assert!(!calls.exists());
16940 let recovery = crate::recovery::assess(
16941 "prior",
16942 Some(&runner.state),
16943 &runner.landing_base(),
16944 crate::recovery::Relation::Diverged,
16945 );
16946 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
16947 let reasons = recovery.reasons.join("\n");
16948 assert!(
16949 reasons.contains(&reviewed) && reasons.contains(&rebased),
16950 "{reasons}"
16951 );
16952 assert!(
16953 reasons.contains("base sync to")
16954 && reasons.contains("additional review is required"),
16955 "{reasons}"
16956 );
16957 }
16958 }
16959 }
16960
16961 #[tokio::test]
16973 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16974 crate::run::pin_test_home();
16975 let home = crate::run::home();
16976
16977 let tmp = tempfile::tempdir().expect("tempdir");
16978 let repo = tmp.path().join("repo");
16979 std::fs::create_dir_all(&repo).unwrap();
16980 init_repo(&repo);
16981 let head = crate::git::rev_parse(&repo, "HEAD")
16982 .await
16983 .expect("rev-parse");
16984 let cache_dir = tmp.path().join("target");
16985
16986 let mut config = Config::default();
16987 config.verify.e2e = vec![format!(
16988 "CARGO_TARGET_DIR='{}' test -f README.md",
16989 cache_dir.display()
16990 )];
16991 config.graph.review_rounds = 1;
16992 config.graph.timeout_verify = Some(2);
16993 config.graph.timeout_resource_wait = 2;
16994
16995 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16996 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16997 .expect("no io error acquiring directly")
16998 {
16999 crate::cache::AcquireOutcome::Acquired(g) => g,
17000 crate::cache::AcquireOutcome::Busy(b) => {
17001 panic!("expected the direct acquire to win the lease first: {b:?}")
17002 }
17003 };
17004
17005 let mut state = RunState::new(
17006 repo.clone(),
17007 "main".to_owned(),
17008 head.clone(),
17009 "task".to_owned(),
17010 config,
17011 );
17012 state.candidates = vec![Candidate {
17013 index: 0,
17014 label: 'A',
17015 agent: "alpha".to_owned(),
17016 branch: "does-not-exist".to_owned(),
17017 worktree: repo.clone(),
17018 summary: String::new(),
17019 stat: String::new(),
17020 files: 0,
17021 commits: 0,
17022 empty: false,
17023 failed: None,
17024 verified_noop: None,
17025 duration_ms: 0,
17026 folded: false,
17027 }];
17028 state.tally = Some(Tally {
17029 first_choice: BTreeMap::from([('A', 1)]),
17030 borda: BTreeMap::new(),
17031 winner: 'A',
17032 rankings: 1,
17033 unanimous_initial: true,
17034 deliberated: false,
17035 changed_votes: 0,
17036 unanimous_final: true,
17037 tie_break: None,
17038 judges: 0,
17039 present: 0,
17040 quorum: 0,
17041 met_quorum: true,
17042 uncontested: Some("only candidate A produced a change".to_owned()),
17043 });
17044 state.reviews = vec![ReviewRound {
17048 round: 1,
17049 head: head.clone(),
17050 verified_head: Some(head.clone()),
17051 verified_at: Some(jiff::Timestamp::now()),
17052 reviews: Vec::new(),
17053 e2e: vec![CommandOutcome {
17054 command: format!(
17055 "CARGO_TARGET_DIR='{}' test -f README.md",
17056 cache_dir.display()
17057 ),
17058 code: None,
17059 output_tail: "waiting for the shared build cache".to_owned(),
17060 duration_ms: 0,
17061 resource_blocked: true,
17062 }],
17063 fix: None,
17064 blocking: 1,
17065 answered: 1,
17066 expected: 1,
17067 clean: false,
17068 verify_retried: false,
17069 e2e_deferred: false,
17070 e2e_defer_reason: None,
17071 progressed: false,
17072 vote_split: false,
17073 reconsideration: Vec::new(),
17074 verdict: None,
17075 }];
17076
17077 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17078 let mut runner = Runner {
17079 state,
17080 roles: ResolvedRoles {
17081 implementers: Vec::new(),
17082 judges: Vec::new(),
17083 reviewers: Vec::new(),
17084 fixer: None,
17085 conductor: conductor(),
17086 implementer_roster: Vec::new(),
17087 judge_roster: Vec::new(),
17088 reviewer_roster: Vec::new(),
17089 },
17090 sem: Arc::new(Semaphore::new(1)),
17091 pause: Pause::new(),
17092 interrupt: Pause::new(),
17093 };
17094
17095 runner.review_loop().await.expect("review_loop");
17100
17101 assert_eq!(
17102 runner.state.reviews.len(),
17103 1,
17104 "no new round was started on top of the unresolved one: {:?}",
17105 runner.state.reviews
17106 );
17107 let last = &runner.state.reviews[0];
17108 assert_eq!(
17109 last.e2e_status(),
17110 E2eStatus::ResourceBlocked,
17111 "still contended: {last:?}"
17112 );
17113 assert!(
17114 last.verified_at.expect("still known") > first_attempt_at,
17115 "review_loop must have actually retried the check, not left it exactly as found"
17116 );
17117 assert_ne!(
17118 runner.state.status,
17119 RunStatus::Blocked,
17120 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17121 runner.state.status
17122 );
17123
17124 held.release();
17125 }
17126
17127 #[tokio::test]
17128 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17129 crate::run::pin_test_home();
17130 for status in [RunStatus::Blocked, RunStatus::Gating] {
17131 let mut runner = runner_at(status);
17132 runner.state.config.graph.land = true;
17133 runner.state.gate_ran = true;
17134 runner.state.merge = Some(MergeOutcome {
17135 mode: MergeMode::Pr,
17136 ok: false,
17137 detail: "the fixer produced no commit".into(),
17138 empty: false,
17139 });
17140 let url = "https://example.invalid/o/r/pull/1";
17141 runner.state.pr = Some(crate::run::PrRecord {
17142 url: url.into(),
17143 number: 1,
17144 state: "open".into(),
17145 checks: "red".into(),
17146 round: 1,
17147 rounds: 1,
17148 red_at_merge: Vec::new(),
17149 });
17150 runner.state.save().unwrap();
17151 runner.execute_graph().await.unwrap();
17155 assert_eq!(runner.state.status, RunStatus::Blocked);
17156 assert!(runner.state.land_sync_pending);
17157 assert!(runner.state.events.iter().any(|event|
17158 event.node == "land" && event.message == format!("watching {url}")));
17159 assert_eq!(
17160 runner.state.merge.as_ref().unwrap().detail,
17161 "the fixer produced no commit"
17162 );
17163 }
17164 }
17165
17166 #[tokio::test]
17167 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17168 crate::run::pin_test_home();
17169 let tmp = tempfile::tempdir().expect("tempdir");
17170 let repo = tmp.path().join("repo");
17171 std::fs::create_dir_all(&repo).unwrap();
17172 init_repo(&repo);
17173
17174 let mut config = Config::default();
17175 config.merge.mode = MergeMode::Pr;
17176 config.graph.land = true;
17177 config.graph.land_approval = false;
17178
17179 let mut state = RunState::new(
17180 repo.clone(),
17181 "main".to_owned(),
17182 "deadbeef".to_owned(),
17183 "task".to_owned(),
17184 config,
17185 );
17186 state.candidates = vec![Candidate {
17187 index: 0,
17188 label: 'A',
17189 agent: "alpha".to_owned(),
17190 branch: "does-not-exist".to_owned(),
17191 worktree: repo.clone(),
17192 summary: String::new(),
17193 stat: String::new(),
17194 files: 0,
17195 commits: 0,
17196 empty: false,
17197 failed: None,
17198 verified_noop: None,
17199 duration_ms: 0,
17200 folded: false,
17201 }];
17202 state.tally = Some(Tally {
17203 first_choice: BTreeMap::from([('A', 1)]),
17204 borda: BTreeMap::new(),
17205 winner: 'A',
17206 rankings: 1,
17207 unanimous_initial: true,
17208 deliberated: false,
17209 changed_votes: 0,
17210 unanimous_final: true,
17211 tie_break: None,
17212 judges: 0,
17213 present: 0,
17214 quorum: 0,
17215 met_quorum: true,
17216 uncontested: Some("only candidate A produced a change".to_owned()),
17217 });
17218 state.reviews = vec![ReviewRound {
17219 round: 1,
17220 head: "deadbeef".to_owned(),
17221 verified_head: None,
17222 verified_at: None,
17223 reviews: Vec::new(),
17224 e2e: Vec::new(),
17225 fix: None,
17226 blocking: 0,
17227 answered: 0,
17228 expected: 0,
17229 clean: true,
17230 verify_retried: false,
17231 e2e_deferred: false,
17232 e2e_defer_reason: None,
17233 progressed: false,
17234 vote_split: false,
17235 reconsideration: Vec::new(),
17236 verdict: None,
17237 }];
17238 state.gate = vec![CommandOutcome {
17239 command: "test".to_owned(),
17240 code: Some(0),
17241 output_tail: String::new(),
17242 duration_ms: 0,
17243 resource_blocked: false,
17244 }];
17245 state.gate_ran = true;
17246 state.status = RunStatus::Landing;
17250 state.merge = Some(MergeOutcome {
17251 mode: MergeMode::Pr,
17252 ok: true,
17253 detail: "https://example.invalid/x/y/pull/1".to_owned(),
17254 empty: false,
17255 });
17256
17257 ask_test_home();
17261 state.save().expect("save landing run");
17264 let store = ask::Questions::open();
17265 let q = ask_open_question(&store, &state.id);
17266
17267 let mut runner = Runner {
17268 state,
17269 roles: ResolvedRoles {
17270 implementers: Vec::new(),
17271 judges: Vec::new(),
17272 reviewers: Vec::new(),
17273 fixer: None,
17274 conductor: conductor(),
17275 implementer_roster: Vec::new(),
17276 judge_roster: Vec::new(),
17277 reviewer_roster: Vec::new(),
17278 },
17279 sem: Arc::new(Semaphore::new(1)),
17280 pause: Pause::new(),
17281 interrupt: Pause::new(),
17282 };
17283
17284 runner.execute().await.expect("execute");
17289
17290 assert_eq!(
17291 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17292 Some("https://example.invalid/x/y/pull/1"),
17293 "reentry must not push again or open a second pull request over the \
17294 one `land` is already watching"
17295 );
17296 assert_ne!(
17297 runner.state.status,
17298 RunStatus::Landing,
17299 "land could not actually reach the fake pull request, so it must \
17300 have given up rather than left the run silently parked forever"
17301 );
17302 assert_eq!(runner.state.status, RunStatus::Blocked);
17306 assert!(
17307 store.get(&q.id).unwrap().status.open(),
17308 "Blocked is still alive; settle_questions must have been a no-op here"
17309 );
17310 }
17311
17312 fn state_with_round(round: ReviewRound) -> RunState {
17313 let mut s = RunState::new(
17314 PathBuf::from("/repo"),
17315 "main".to_owned(),
17316 "abc1234".to_owned(),
17317 "add retries".to_owned(),
17318 Config::default(),
17319 );
17320 s.reviews = vec![round];
17321 s
17322 }
17323
17324 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17325 crate::verdict::Finding {
17326 id: id.to_owned(),
17327 severity,
17328 file: None,
17329 line: None,
17330 title: title.to_owned(),
17331 detail: String::new(),
17332 }
17333 }
17334
17335 #[test]
17336 fn pr_body_names_open_findings_and_declined_ones() {
17337 let round = ReviewRound {
17338 round: 2,
17339 head: "deadbee".to_owned(),
17340 verified_head: None,
17341 verified_at: None,
17342 reviews: vec![ReviewRecord {
17343 attempts: 0,
17344 reviewer: 1,
17345 agent: "alpha".to_owned(),
17346 summary: String::new(),
17347 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17348 vote: None,
17349 failed: None,
17350 duration_ms: 0,
17351 }],
17352 e2e: vec![CommandOutcome {
17353 command: "cargo test".to_owned(),
17354 code: Some(0),
17355 output_tail: String::new(),
17356 duration_ms: 0,
17357 resource_blocked: false,
17358 }],
17359 verify_retried: false,
17360 e2e_deferred: false,
17361 e2e_defer_reason: None,
17362 fix: Some(FixRecord {
17363 agent: "alpha".to_owned(),
17364 addressed: Vec::new(),
17365 rejected: vec![crate::verdict::Rejection {
17366 id: "R1-1-1".to_owned(),
17367 why: "not reachable from any caller".to_owned(),
17368 }],
17369 notes: String::new(),
17370 committed: true,
17371 failed: None,
17372 duration_ms: 0,
17373 continuation: None,
17374 }),
17375 blocking: 0,
17376 answered: 1,
17377 expected: 1,
17378 clean: false,
17379 progressed: true,
17380 vote_split: false,
17381 reconsideration: Vec::new(),
17382 verdict: None,
17383 };
17384 let state = state_with_round(round);
17385 let body = pr_message(&state, 'A').body;
17386
17387 assert!(body.contains("add retries"), "the task must still be there");
17388 assert!(body.contains("R2-1-1"), "{body}");
17389 assert!(!body.contains("unused import"), "{body}");
17390 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17391 assert!(
17392 !body.contains("not reachable from any caller"),
17393 "the reason it was declined: {body}"
17394 );
17395 }
17396
17397 #[test]
17398 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17399 let round = ReviewRound {
17400 round: 1,
17401 head: "deadbee".to_owned(),
17402 verified_head: None,
17403 verified_at: None,
17404 reviews: vec![ReviewRecord {
17405 attempts: 0,
17406 reviewer: 1,
17407 agent: "alpha".to_owned(),
17408 summary: String::new(),
17409 findings: Vec::new(),
17410 vote: None,
17411 failed: None,
17412 duration_ms: 0,
17413 }],
17414 e2e: Vec::new(),
17415 verify_retried: false,
17416 e2e_deferred: false,
17417 e2e_defer_reason: None,
17418 fix: None,
17419 blocking: 0,
17420 answered: 1,
17421 expected: 1,
17422 clean: true,
17423 progressed: false,
17424 vote_split: false,
17425 reconsideration: Vec::new(),
17426 verdict: None,
17427 };
17428 let state = state_with_round(round);
17429 let body = pr_message(&state, 'A').body;
17430 assert!(!body.contains("Open review findings"), "{body}");
17431 assert!(!body.contains("Declined"), "{body}");
17432 }
17433
17434 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17435 let mut state = RunState::new(
17436 PathBuf::from("/repo"),
17437 "main".to_owned(),
17438 "abc1234".to_owned(),
17439 instruction.to_owned(),
17440 Config::default(),
17441 );
17442 state.candidates.push(Candidate {
17443 index: 0,
17444 label: 'A',
17445 agent: "alpha".to_owned(),
17446 branch: "magi/x/A".to_owned(),
17447 worktree: PathBuf::from("/wt"),
17448 summary: summary.to_owned(),
17449 stat: String::new(),
17450 files: 1,
17451 commits: 1,
17452 empty: false,
17453 failed: None,
17454 verified_noop: None,
17455 folded: false,
17456 duration_ms: 0,
17457 });
17458 state
17459 }
17460
17461 fn review_state(subjects: &[&str]) -> RunState {
17462 let mut state = state_with_summary(
17463 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17464 "",
17465 );
17466 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17467 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17468 state
17469 }
17470
17471 #[tokio::test]
17474 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17475 ask_test_home();
17476 let tmp = tempfile::tempdir().unwrap();
17477 let repo = tmp.path().join("repo");
17478 std::fs::create_dir_all(&repo).unwrap();
17479 init_repo(&repo);
17480 let origin = tmp.path().join("origin.git");
17481 let g = |dir: &Path, args: &[&str]| {
17482 let out = std::process::Command::new("git")
17483 .args(args)
17484 .current_dir(dir)
17485 .quiet()
17486 .output()
17487 .expect("spawn git");
17488 assert!(
17489 out.status.success(),
17490 "git {args:?}: {}",
17491 String::from_utf8_lossy(&out.stderr)
17492 );
17493 };
17494 g(
17495 tmp.path(),
17496 &[
17497 "clone",
17498 "--bare",
17499 "-q",
17500 repo.to_str().unwrap(),
17501 origin.to_str().unwrap(),
17502 ],
17503 );
17504 g(
17505 &repo,
17506 &["remote", "add", "origin", origin.to_str().unwrap()],
17507 );
17508 let c1 = git::rev_parse(&repo, "main").await.unwrap();
17509
17510 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17512 g(&repo, &["add", "-A"]);
17513 g(
17514 &repo,
17515 &["commit", "-q", "-m", "chore(deps): update a crate"],
17516 );
17517 g(&repo, &["push", "-q", "origin", "main"]);
17518 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17519 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17520 g(&repo, &["add", "-A"]);
17521 g(
17522 &repo,
17523 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17524 );
17525 g(&repo, &["checkout", "-q", "main"]);
17526
17527 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17528 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17529 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17531 assert_eq!(fallback, c1);
17532
17533 let mut state = review_state(&[
17534 "chore(deps): update a crate",
17535 "fix(daemon): apply a chosen action",
17536 ]);
17537 state.repo = repo.clone();
17538 state.base_branch = "main".to_owned();
17539 state.base_commit = c1;
17540 refresh_reviewed_commits(&mut state, "feat/own").await;
17541 assert_eq!(
17542 state.reviewed_commits,
17543 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17544 );
17545 assert_eq!(
17546 review_title(&state).as_deref(),
17547 Some("fix(daemon): apply a chosen action")
17548 );
17549 assert_eq!(
17550 leaked_subjects(&state, "feat/own").await,
17551 Some(vec!["chore(deps): update a crate".to_owned()])
17552 );
17553 state.config.merge.remote = "nowhere".to_owned();
17556 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17557 }
17558
17559 struct SummaryFixture {
17563 _tmp: tempfile::TempDir,
17564 repo: PathBuf,
17565 reply: PathBuf,
17566 runner: Runner,
17567 }
17568
17569 fn summary_fixture() -> SummaryFixture {
17570 ask_test_home();
17571 let tmp = tempfile::tempdir().unwrap();
17572 let repo = tmp.path().join("repo");
17573 std::fs::create_dir_all(&repo).unwrap();
17574 init_repo(&repo);
17575 let origin = tmp.path().join("origin.git");
17576 let g = |args: &[&str]| {
17577 let out = std::process::Command::new("git")
17578 .args(args)
17579 .current_dir(&repo)
17580 .quiet()
17581 .output()
17582 .expect("spawn git");
17583 assert!(
17584 out.status.success(),
17585 "git {args:?}: {}",
17586 String::from_utf8_lossy(&out.stderr)
17587 );
17588 };
17589 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17590 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17591 g(&["push", "-q", "origin", "main"]);
17592 g(&["checkout", "-q", "-b", "feat/own"]);
17593 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17594 g(&["add", "-A"]);
17595 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17596 let reply = tmp.path().join("reply.txt");
17597 std::fs::write(
17598 &reply,
17599 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17600 )
17601 .unwrap();
17602 let mut runner = runner_at(RunStatus::Gating);
17603 let mut state = review_state(&["feat(own): add the own file"]);
17604 state.repo = repo.clone();
17605 state.base_branch = "main".to_owned();
17606 state.candidates[0].branch = "feat/own".to_owned();
17607 state.candidates[0].worktree = repo.clone();
17608 let mut author = spec("alpha");
17609 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17610 author
17611 .env
17612 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17613 state.config.agents = vec![author];
17614 runner.state = state;
17615 SummaryFixture {
17616 _tmp: tmp,
17617 repo,
17618 reply,
17619 runner,
17620 }
17621 }
17622
17623 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17624 let winner = f.runner.state.candidates[0].clone();
17625 f.runner
17626 .guarded_pr_message(&winner, None, true)
17627 .await
17628 .unwrap()
17629 .expect("an English description never parks")
17630 }
17631
17632 fn clean_round_with_minor_findings() -> ReviewRound {
17633 let mut round = passing_round(2, None);
17634 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17635 round.reviews = vec![ReviewRecord {
17636 attempts: 0,
17637 reviewer: 1,
17638 agent: "alpha".into(),
17639 summary: "内部レビュー".into(),
17640 findings: vec![
17641 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17642 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17643 ],
17644 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17645 failed: None,
17646 duration_ms: 0,
17647 }];
17648 round
17649 }
17650
17651 #[tokio::test]
17652 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17653 for review_only in [false, true] {
17654 let mut f = summary_fixture();
17655 if !review_only {
17656 f.runner.state.candidates[0].agent = "alpha".into();
17657 }
17658 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17659 assert!(f.runner.state.reviews[0].clean);
17660 assert!(f.runner.state.reviews[0].fix.is_none());
17661 assert!(f.runner.state.open_findings().is_empty());
17662 assert_eq!(
17663 crate::pr_outcome::required_finding_ids(&f.runner.state),
17664 ["R2-1-1", "R2-1-2"]
17665 );
17666 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17667 let evidence = crate::pr_outcome::summary_evidence(
17668 &f.runner.state,
17669 crate::pr_outcome::FinalHead::At(&head),
17670 );
17671 assert!(
17672 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17673 );
17674 let omitted = posted(&mut f).await;
17676 assert!(omitted.body.contains("No description"));
17677 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17678 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17679 assert!(f.runner.state.change_summary.is_none());
17680 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17681 let described = posted(&mut f).await;
17682 assert!(described.body.contains("improve documentation for callers"));
17683 assert!(described.body.contains("remaining edge case"));
17684 assert!(described.body.contains("## Open review findings"));
17685 assert!(!described.body.contains("未修正"));
17686 }
17687 }
17688
17689 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17690 let mut f = summary_fixture();
17691 if !review_only {
17692 f.runner.state.candidates[0].agent = "alpha".into();
17693 }
17694 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17695 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17696 fixed
17697 .fix
17698 .as_mut()
17699 .unwrap()
17700 .rejected
17701 .push(crate::verdict::Rejection {
17702 id: "R1-1-1".into(),
17703 why: "呼び出されないコードのため辞退".into(),
17704 });
17705 let mut clean = clean_round_with_minor_findings();
17706 clean.verified_head = Some(head.clone());
17707 f.runner.state.reviews = vec![fixed, clean];
17708 ok_gate(&mut f.runner.state, Some(&head));
17709 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17710 f.runner.state.config.agents.push(scripted("beta", reply));
17711 f.runner.state.config.roles.language_judge =
17712 Some(crate::config::AgentChoice::One("beta".into()));
17713 f
17714 }
17715
17716 #[tokio::test]
17717 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17718 for review_only in [false, true] {
17719 let mut f = rewrite_fixture_with_findings(review_only,
17720 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17721 let m = posted(&mut f).await;
17722 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17723 let expected =
17724 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17725 assert!(m.body.contains(&expected));
17726 assert!(m.body.contains("## Review rounds"));
17727 assert!(
17728 m.body
17729 .contains("Gate (`verify.gate`): passed on the final commit")
17730 );
17731 assert!(
17732 m.body
17733 .contains("Tests (`verify.e2e`): passed on the final commit")
17734 );
17735 let c = f.runner.state.change_summary.as_ref().unwrap();
17736 assert!(
17737 m.body
17738 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17739 );
17740 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17741 assert!(crate::pr_outcome::covers_required_findings(
17742 &f.runner.state,
17743 &m.body
17744 ));
17745 assert!(m.body.contains("documentation remains incomplete"));
17746 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17747 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17748 assert!(f.runner.state.github_text.is_none());
17749 }
17750 }
17751
17752 #[tokio::test]
17753 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17754 for review_only in [false, true] {
17755 for reply in [
17756 "TITLE: fix: public description\nA short English description.",
17757 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17758 ] {
17759 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17760 let winner = f.runner.state.candidates[0].clone();
17761 let posted = f
17762 .runner
17763 .guarded_pr_message(&winner, None, true)
17764 .await
17765 .unwrap();
17766 assert!(posted.is_none());
17767 assert!(f.runner.state.parked);
17768 let withheld = f.runner.state.github_text.as_ref().unwrap();
17769 assert!(
17770 withheld.rewrite.is_none(),
17771 "an incomplete rewrite must not be selectable"
17772 );
17773 let q = ask::Questions::open()
17774 .get(withheld.question.as_ref().unwrap())
17775 .unwrap();
17776 assert!(
17777 !q.choices
17778 .contains(&crate::github_text::USE_REWRITE.to_owned())
17779 );
17780 assert!(
17781 q.detail.contains("R1-1-1")
17782 && q.detail.contains("R2-1-1")
17783 && q.detail.contains("R2-1-2")
17784 );
17785 }
17786 }
17787 }
17788
17789 #[tokio::test]
17790 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17791 for review_only in [false, true] {
17792 let mut f = summary_fixture();
17793 if !review_only {
17794 f.runner.state.candidates[0].agent = "alpha".into();
17795 }
17796 f.runner.state.instruction = "内部タスクの説明".into();
17797 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17798 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17799 f.runner.state.reviews = vec![
17800 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17801 round_with_notes(2, Some("二回目の内部修正報告です。")),
17802 passing_round(3, Some(&head)),
17803 ];
17804 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17805 ok_gate(&mut f.runner.state, Some(&head));
17806 let captured = f.repo.join("public-prompt.txt");
17807 let author = &mut f.runner.state.config.agents[0];
17808 author.command = vec![
17809 "sh".into(),
17810 "-c".into(),
17811 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17812 ];
17813 author.env.insert(
17814 "PROMPT_FILE".into(),
17815 captured.to_string_lossy().into_owned(),
17816 );
17817 let m = posted(&mut f).await;
17818 assert!(m.body.contains("Adds `own.txt`"));
17819 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17820 assert!(m.body.contains("R1-1-1"));
17821 assert!(
17822 m.body
17823 .contains("Gate (`verify.gate`): passed on the final commit")
17824 );
17825 assert!(
17826 m.body
17827 .contains("Tests (`verify.e2e`): passed on the final commit")
17828 );
17829 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17830 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17831 let prompt = std::fs::read_to_string(captured).unwrap();
17832 assert!(prompt.contains(&format!(
17833 "git diff {} {head}",
17834 f.runner.state.change_summary.as_ref().unwrap().base
17835 )));
17836 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17837 assert_eq!(
17838 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17839 "二回目の内部修正報告です。"
17840 );
17841 }
17842 }
17843
17844 #[tokio::test]
17845 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17846 for review_only in [false, true] {
17847 let mut f = summary_fixture();
17848 if !review_only {
17849 f.runner.state.candidates[0].agent = "alpha".into();
17850 }
17851 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17852 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17853 std::fs::write(&f.reply, "no usable description").unwrap();
17854 let failed = posted(&mut f).await;
17855 assert!(
17856 failed
17857 .body
17858 .contains("No description of what this change does could be generated")
17859 );
17860 assert!(failed.body.is_ascii());
17861 assert!(f.runner.state.github_text.is_none());
17862 assert!(f.runner.state.change_summary.is_none());
17863 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17864 let evidence = crate::pr_outcome::summary_evidence(
17865 &f.runner.state,
17866 crate::pr_outcome::FinalHead::At(&head),
17867 );
17868 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17869 assert!(
17870 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17871 .exists()
17872 );
17873 f.runner.state =
17874 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17875 std::fs::write(
17876 &f.reply,
17877 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17878 )
17879 .unwrap();
17880 let retried = posted(&mut f).await;
17881 assert!(retried.body.contains("Adds the committed own file."));
17882 assert_eq!(
17883 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17884 "日本語の修正報告"
17885 );
17886 }
17887 }
17888
17889 #[tokio::test]
17890 async fn same_head_verification_and_findings_invalidate_public_prose() {
17891 let mut f = summary_fixture();
17892 posted(&mut f).await;
17893 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17894 let original = f
17895 .runner
17896 .state
17897 .change_summary
17898 .as_ref()
17899 .unwrap()
17900 .evidence
17901 .clone();
17902 ok_gate(&mut f.runner.state, Some(&head));
17903 std::fs::write(
17904 &f.reply,
17905 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17906 )
17907 .unwrap();
17908 let verified = posted(&mut f).await;
17909 assert!(verified.body.contains("now verified"));
17910 assert_ne!(
17911 f.runner.state.change_summary.as_ref().unwrap().evidence,
17912 original
17913 );
17914 let mut round = round_with_notes(1, Some("内部報告"));
17915 round.reviews.push(ReviewRecord {
17916 attempts: 0,
17917 reviewer: 1,
17918 agent: "alpha".into(),
17919 summary: "内部レビュー".into(),
17920 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17921 vote: None,
17922 failed: None,
17923 duration_ms: 0,
17924 });
17925 round
17926 .fix
17927 .as_mut()
17928 .unwrap()
17929 .rejected
17930 .push(crate::verdict::Rejection {
17931 id: "R1-1-1".into(),
17932 why: "呼び出し元から到達できない".into(),
17933 });
17934 f.runner.state.reviews = vec![round];
17935 let omitted = posted(&mut f).await;
17937 assert!(omitted.body.contains("No description"));
17938 assert!(omitted.body.contains("R1-1-1"));
17939 assert!(!omitted.body.contains("到達"));
17940 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17941 let described = posted(&mut f).await;
17942 assert!(described.body.contains("unreachable from callers"));
17943 assert!(described.body.contains("R1-1-1"));
17944 assert!(!described.body.contains("到達"));
17945 let evidence = f
17946 .runner
17947 .state
17948 .change_summary
17949 .as_ref()
17950 .unwrap()
17951 .evidence
17952 .clone();
17953 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17954 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17955 assert!(posted(&mut f).await.body.contains("rationale updated"));
17956 assert_ne!(
17957 f.runner.state.change_summary.as_ref().unwrap().evidence,
17958 evidence
17959 );
17960 f.runner
17962 .state
17963 .change_summary
17964 .as_mut()
17965 .unwrap()
17966 .evidence
17967 .clear();
17968 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17969 assert!(posted(&mut f).await.body.contains("current specification"));
17970 }
17971
17972 #[tokio::test]
17973 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17974 let mut f = summary_fixture();
17975 let m = posted(&mut f).await;
17976 assert_eq!(m.title, "feat(own): add the own file");
17977 assert!(
17978 m.body
17979 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17980 "{}",
17981 m.body
17982 );
17983 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17984 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17985 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17986 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17987 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17988 let kept = f.runner.state.change_summary.as_ref().unwrap();
17989 assert_eq!(
17990 (kept.head.as_str(), kept.base.as_str()),
17991 (head.as_str(), base.as_str())
17992 );
17993 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17994 }
17995
17996 #[tokio::test]
17997 async fn review_only_summary_receives_the_configured_github_language() {
17998 let mut f = summary_fixture();
17999 f.runner.state.config.graph.github_language = "ja".into();
18000 let captured = f.repo.join("summary-prompt.txt");
18001 let author = &mut f.runner.state.config.agents[0];
18002 author.command = vec![
18003 "sh".into(),
18004 "-c".into(),
18005 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18006 ];
18007 author.env.insert(
18008 "PROMPT_FILE".into(),
18009 captured.to_string_lossy().into_owned(),
18010 );
18011 std::fs::write(
18012 &f.reply,
18013 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18014 )
18015 .unwrap();
18016 let m = posted(&mut f).await;
18017 let prompt = std::fs::read_to_string(captured).unwrap();
18018 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18019 assert!(!prompt.contains("in English"));
18020 assert_eq!(m.title, "feat(own): ファイルを追加");
18021 assert!(m.body.contains("own.txt を追加します。"));
18022 }
18023
18024 #[tokio::test]
18025 async fn review_fixes_come_after_the_whole_change_summary() {
18026 let mut f = summary_fixture();
18027 f.runner.state.reviews = vec![
18028 round_with_notes(1, Some("handled the PATH case")),
18029 round_with_notes(2, None),
18030 ];
18031 let m = posted(&mut f).await;
18032 let summary = m.body.find("## Summary").expect("summary");
18033 let fixes = m.body.find("## Review rounds").expect("review rounds");
18034 assert!(summary < fixes, "{}", m.body);
18035 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18036 assert!(m.body.contains("## Final verification"), "{}", m.body);
18037 }
18038
18039 #[tokio::test]
18040 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18041 let mut f = summary_fixture();
18042 let first = posted(&mut f).await;
18043 let first_head = f.runner.state.change_summary.clone().unwrap().head;
18044 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18046 let again = posted(&mut f).await;
18047 assert_eq!((again.title, again.body), (first.title, first.body));
18048
18049 let g = |args: &[&str]| {
18050 let out = std::process::Command::new("git")
18051 .args(args)
18052 .current_dir(&f.repo)
18053 .quiet()
18054 .output()
18055 .expect("spawn git");
18056 assert!(out.status.success(), "git {args:?}");
18057 };
18058 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18059 g(&["add", "-A"]);
18060 g(&["commit", "-q", "-m", "fix(own): handle more"]);
18061 let second = posted(&mut f).await;
18062 let kept = f.runner.state.change_summary.clone().unwrap();
18063 assert_ne!(kept.head, first_head);
18064 assert!(second.body.contains("Not read yet."), "{}", second.body);
18065 assert!(!second.body.contains("so the first summary is read"));
18066 assert_eq!(second.title, "feat(own): other");
18067 }
18068
18069 #[tokio::test]
18070 async fn an_unavailable_summary_is_stated_and_never_invented() {
18071 let mut f = summary_fixture();
18072 std::fs::write(&f.reply, "no title line here").unwrap();
18073 let m = posted(&mut f).await;
18074 assert!(
18075 m.body
18076 .contains("No description of what this change does could be generated"),
18077 "{}",
18078 m.body
18079 );
18080 assert!(m.body.contains("internal reports remain in local records"));
18081 assert!(f.runner.state.change_summary.is_none());
18082 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18083 let winner = f.runner.state.candidates[0].clone();
18085 let preview = f
18086 .runner
18087 .guarded_pr_message(&winner, None, false)
18088 .await
18089 .unwrap()
18090 .unwrap();
18091 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18092 }
18093
18094 #[test]
18095 fn pr_message_review_single_commit_uses_its_subject() {
18096 let state = review_state(&["feat(nats): per-role user"]);
18097 let m = pr_message(&state, 'A');
18098 assert_eq!(m.title, "feat(nats): per-role user");
18099 assert!(!m.body.contains("Review the work already"), "{}", m.body);
18100 assert!(m.body.contains("## Commits under review"), "{}", m.body);
18101 }
18102
18103 #[test]
18104 fn pr_message_review_multi_commit_takes_the_oldest() {
18105 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18106 let m = pr_message(&state, 'A');
18107 assert_eq!(m.title, "feat: the change");
18108 for s in ["feat: the change", "fix: typo", "fix: again"] {
18109 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18110 }
18111 }
18112
18113 #[test]
18114 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18115 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18116 let mut state = review_state(&[first, "fix: later fixup"]);
18117 state.candidates[0].branch = "機能/ログイン".to_owned();
18118 let m = pr_message(&state, 'A');
18119 assert!(
18120 m.title.starts_with("chore: land candidate A of run"),
18121 "{}",
18122 m.title
18123 );
18124 }
18125 }
18126
18127 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18128 BranchFacts {
18129 commits: commits
18130 .iter()
18131 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18132 .collect(),
18133 stat: stat.to_owned(),
18134 }
18135 }
18136
18137 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18138 let mut r = review_round(true, 0, 1, 1, true, true);
18139 r.round = round;
18140 r.fix = notes.map(|n| FixRecord {
18141 agent: "fixer".to_owned(),
18142 addressed: Vec::new(),
18143 rejected: Vec::new(),
18144 notes: n.to_owned(),
18145 committed: true,
18146 failed: None,
18147 duration_ms: 0,
18148 continuation: None,
18149 });
18150 r
18151 }
18152
18153 #[test]
18154 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18155 let state = review_state(&["ignored"]);
18156 let f = facts(
18157 &[
18158 (
18159 "fix(login): resolve PATH on macOS",
18160 "Login shells skip rc files.",
18161 ),
18162 ("fix: address review", ""),
18163 ],
18164 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18165 );
18166 let m = pr_message_with(&state, 'A', Some(&f));
18167 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18168 assert!(
18169 m.body
18170 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
18171 "{}",
18172 m.body
18173 );
18174 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18175 assert!(m.body.contains("## Diff stat"), "{}", m.body);
18176 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18177 for banned in [
18178 "Review the work already",
18179 "no task statement",
18180 "Original task",
18181 ] {
18182 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18183 }
18184 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18185 }
18186
18187 #[test]
18188 fn pr_message_review_truncates_a_huge_first_commit_body() {
18189 let state = review_state(&["ignored"]);
18190 let f = facts(
18191 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18192 "s",
18193 );
18194 let m = pr_message_with(&state, 'A', Some(&f));
18195 assert!(m.body.len() < 30_000, "{}", m.body.len());
18196 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18197 assert!(
18198 m.body.contains("- ... 1 more commit(s)"),
18199 "{}",
18200 m.body.len()
18201 );
18202 assert!(m.body.ends_with("magi:candidate-a\n"));
18203 }
18204
18205 #[test]
18206 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18207 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18208 assert_eq!(m.title, "feat: x");
18209 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18210 assert!(!m.body.contains("Diff stat"), "{}", m.body);
18211 assert!(!m.body.contains("no task statement"), "{}", m.body);
18212 }
18213
18214 #[test]
18215 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18216 let mut state = review_state(&["日本語の件名"]);
18217 state.candidates[0].branch = "fix/macos-login-path".to_owned();
18218 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18219 state.candidates[0].branch = "機能/ログイン".to_owned();
18220 assert!(
18221 pr_message(&state, 'A')
18222 .title
18223 .starts_with("chore: land candidate A")
18224 );
18225 }
18226
18227 #[test]
18228 fn pr_message_review_fixes_survive_a_clean_final_round() {
18229 let mut state = review_state(&["feat: x"]);
18230 state.reviews = vec![
18231 round_with_notes(1, Some("handled the PATH case")),
18232 round_with_notes(2, None),
18233 ];
18234 let body = pr_message(&state, 'A').body;
18235 assert!(
18236 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18237 "{body}"
18238 );
18239 assert!(body.contains("### Round 1"), "{body}");
18240
18241 state.reviews = vec![
18242 round_with_notes(1, Some("first fix")),
18243 round_with_notes(2, Some("")),
18244 round_with_notes(3, Some("second fix")),
18245 round_with_notes(4, None),
18246 ];
18247 let body = pr_message(&state, 'A').body;
18248 assert!(
18249 body.contains("### Round 1") && !body.contains("first fix"),
18250 "{body}"
18251 );
18252 assert!(
18253 body.contains("### Round 3") && !body.contains("second fix"),
18254 "{body}"
18255 );
18256 assert!(body.contains("### Round 2"), "{body}");
18257 }
18258
18259 const FINAL: &str = "abcdef1234567890";
18260
18261 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18262 let mut r = round_with_notes(round, None);
18263 r.verified_head = verified.map(str::to_owned);
18264 r
18265 }
18266
18267 fn ok_gate(state: &mut RunState, head: Option<&str>) {
18268 state.gate = vec![CommandOutcome {
18269 command: "gate".to_owned(),
18270 code: Some(0),
18271 output_tail: String::new(),
18272 duration_ms: 0,
18273 resource_blocked: false,
18274 }];
18275 state.gate_ran = true;
18276 state.gate_head = head.map(str::to_owned);
18277 }
18278
18279 fn body_at(state: &RunState) -> String {
18280 pr_message_raw(
18281 state,
18282 'A',
18283 None,
18284 crate::pr_outcome::FinalHead::At(FINAL),
18285 ChangeNote::NotAsked,
18286 )
18287 .body
18288 }
18289
18290 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18291
18292 #[test]
18293 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18294 let mut state = review_state(&["feat: x"]);
18295 state.reviews = vec![
18296 round_with_notes(1, Some(SANDBOXED)),
18297 passing_round(2, Some(FINAL)),
18298 ];
18299 ok_gate(&mut state, Some(FINAL));
18300 let body = body_at(&state);
18301 assert!(!body.contains("Handled the PATH case."), "{body}");
18302 assert!(!body.contains("could not commit"), "{body}");
18303 assert!(!body.contains("could not run"), "{body}");
18304 assert!(
18305 body.contains("The fix is committed on the branch"),
18306 "{body}"
18307 );
18308 assert!(!body.contains("sandbox"), "{body}");
18309 assert!(
18310 body.contains("Tests (`verify.e2e`): passed on the final commit"),
18311 "{body}"
18312 );
18313 assert!(
18314 body.contains("Gate (`verify.gate`): passed on the final commit"),
18315 "{body}"
18316 );
18317 assert!(
18319 state.reviews[0]
18320 .fix
18321 .as_ref()
18322 .unwrap()
18323 .notes
18324 .contains("could not commit")
18325 );
18326 }
18327
18328 #[test]
18329 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18330 let mut state = review_state(&["feat: x"]);
18331 let mut r1 = round_with_notes(1, Some(SANDBOXED));
18332 r1.fix.as_mut().unwrap().committed = false;
18333 let mut r2 = passing_round(2, Some(FINAL));
18334 r2.e2e[0].code = Some(1);
18335 state.reviews = vec![r1, r2];
18336 let body = body_at(&state);
18337 assert!(!body.contains("could not commit"), "{body}");
18338 assert!(!body.contains("could not run"), "{body}");
18339 assert!(body.contains("This round produced no commit"), "{body}");
18340 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18341 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18342 assert!(!body.contains("passed"), "{body}");
18343 }
18344
18345 #[test]
18346 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18347 let mut state = review_state(&["feat: x"]);
18348 state.reviews = vec![
18349 round_with_notes(1, Some(SANDBOXED)),
18350 passing_round(2, Some("0123456789abcdef")),
18351 ];
18352 ok_gate(&mut state, Some("0123456789abcdef"));
18353 let body = body_at(&state);
18354 assert!(!body.contains("HTTP tests could not run"), "{body}");
18355 assert!(!body.contains("could not commit"), "{body}");
18357 assert!(
18358 body.contains("passed on an earlier commit (`0123456`)"),
18359 "{body}"
18360 );
18361 assert!(!body.contains("passed on the final commit"), "{body}");
18362
18363 ok_gate(&mut state, None);
18365 let body = body_at(&state);
18366 assert!(
18367 body.contains("passed on a commit that was not recorded"),
18368 "{body}"
18369 );
18370 }
18371
18372 #[test]
18373 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18374 let state = state_with_summary(
18375 "add retries to the client",
18376 "TITLE: feat: retries\n\nDid it.",
18377 );
18378 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18379 assert_eq!(m.title, "feat: retries");
18380 assert!(
18381 m.body.contains("<summary>Original task</summary>"),
18382 "{}",
18383 m.body
18384 );
18385 assert!(!m.body.contains("Commits under review"), "{}", m.body);
18386 assert!(
18387 m.body
18388 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18389 "{}",
18390 m.body
18391 );
18392 }
18393
18394 #[test]
18395 fn pr_message_review_bounds_a_long_english_subject() {
18396 let long = format!("feat: {}", "word ".repeat(100));
18397 let m = pr_message(&review_state(&[&long]), 'A');
18398 assert!(m.title.starts_with("feat: word"), "{}", m.title);
18399 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18400 }
18401
18402 #[test]
18403 fn pr_message_implementation_run_is_unchanged_by_review_support() {
18404 let state = state_with_summary("add retries\n\ndetails", "- did some things");
18405 let m = pr_message(&state, 'A');
18406 assert_eq!(m.title, "add retries");
18407 assert!(m.body.contains("<summary>Original task</summary>"));
18408 assert!(!m.body.contains("Commits under review"));
18409 assert_eq!(landing_subject_source(&state), state.instruction);
18410 }
18411
18412 #[test]
18413 fn review_run_squash_subject_is_the_change_not_the_prompt() {
18414 let state = review_state(&["feat: the change", "fix: typo"]);
18415 let source = landing_subject_source(&state);
18416 assert_eq!(land::merge_subject("", &source), "feat: the change");
18417 assert_eq!(
18418 land::merge_subject("magi: candidate A (uncommitted work)", &source),
18419 "feat: the change"
18420 );
18421 assert_eq!(
18423 land::merge_subject("feat: renamed by hand", &source),
18424 "feat: renamed by hand"
18425 );
18426 let blank = review_state(&["日本語"]);
18427 assert!(
18428 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18429 );
18430 }
18431
18432 #[test]
18433 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18434 let state = review_state(&["feat: the change"]);
18435 let source = landing_subject_source(&state);
18436 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18437 assert_eq!(
18438 land::merge_subject(landing_title(&state, old), &source),
18439 "feat: the change"
18440 );
18441 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18442 let task = state_with_summary("add retries", "");
18443 assert_eq!(landing_title(&task, old), old);
18444 }
18445
18446 #[test]
18447 fn pr_message_describes_the_change_not_the_task() {
18448 let state = state_with_summary(
18449 "今回やってほしいこと: results projector を直す",
18450 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18451 );
18452 let m = pr_message(&state, 'A');
18453 assert_eq!(m.title, "fix(web): batch the runs list reads");
18454 assert!(
18455 m.body.starts_with("## Summary\n\n- reads run.json once"),
18456 "{}",
18457 m.body
18458 );
18459 assert!(!m.body.contains("TITLE:"), "{}", m.body);
18460 let task_at = m.body.find("今回やってほしいこと").unwrap();
18461 let details_at = m.body.find("<details>").unwrap();
18462 assert!(
18463 details_at < task_at,
18464 "the task lives inside <details>: {}",
18465 m.body
18466 );
18467 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18468 assert!(m.body.contains("magi:candidate-a"));
18469 }
18470
18471 #[test]
18472 fn pr_message_falls_back_to_the_task_without_a_title_line() {
18473 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18474 let m = pr_message(&state, 'A');
18475 assert_eq!(m.title, "add retries");
18476 assert!(
18477 m.body.contains("## Summary\n\n- did some things"),
18478 "{}",
18479 m.body
18480 );
18481
18482 let none = RunState::new(
18483 PathBuf::from("/repo"),
18484 "main".to_owned(),
18485 "abc1234".to_owned(),
18486 "add retries".to_owned(),
18487 Config::default(),
18488 );
18489 let m = pr_message(&none, 'A');
18490 assert_eq!(m.title, "add retries");
18491 assert!(!m.body.contains("## Summary"), "{}", m.body);
18492 }
18493
18494 #[test]
18495 fn pr_message_refuses_the_candidate_commit_subject() {
18496 for bad in [
18497 "TITLE: magi: candidate A (uncommitted work)",
18498 "TITLE: chore: stuff (uncommitted work)",
18499 "TITLE: ",
18500 ] {
18501 let state = state_with_summary("add retries", bad);
18502 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18503 }
18504 }
18505
18506 #[test]
18507 fn pr_message_bounds_a_very_long_task_and_title() {
18508 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18509 let state = state_with_summary(&long, "- nothing");
18510 let m = pr_message(&state, 'A');
18511 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18512 assert!(!m.title.contains('\n'));
18513
18514 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18515 let m = pr_message(&state, 'A');
18516 assert!(m.title.starts_with("feat: "));
18517 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18518 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18519 }
18520
18521 fn long_title_of(instruction: &str) -> String {
18522 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18523 }
18524
18525 #[test]
18526 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18527 let first = "Make the landing path keep a readable title for long tasks";
18528 let line = format!(
18529 "{first}. {}",
18530 "And then keep going with more words ".repeat(20)
18531 );
18532 let t = long_title_of(&line);
18533 assert_eq!(t, first);
18534 assert!(!t.starts_with("chore: land"));
18535 }
18536
18537 #[test]
18538 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18539 let line = "word ".repeat(200);
18540 let t = long_title_of(&line);
18541 assert!(t.ends_with("word..."), "{t}");
18542 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18543 }
18544
18545 #[test]
18546 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18547 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18548 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18549 }
18550 }
18551
18552 #[test]
18553 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18554 let mut state = state_with_summary("再試行を追加する", "- nothing");
18555 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18556 state.config.graph.github_language = "ja".into();
18557 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18558 }
18559
18560 #[test]
18561 fn pr_message_title_limit_is_exact() {
18562 let at = "a".repeat(PR_TITLE_MAX);
18563 assert_eq!(long_title_of(&at), at);
18564 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18565 assert!(over.ends_with("..."), "{over}");
18566 assert_eq!(over.chars().count(), PR_TITLE_MAX);
18567 }
18568
18569 #[test]
18570 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18571 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18572 let t = long_title_of(&line);
18573 assert!(t.ends_with("..."), "{t}");
18574 assert!(t.is_ascii(), "{t}");
18575 }
18576
18577 #[test]
18578 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18579 let line = format!(
18580 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18581 "plus more filler words ".repeat(20)
18582 );
18583 let t = long_title_of(&line);
18584 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18585 }
18586
18587 #[test]
18588 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18589 let line = format!("\n\n# {}", "title words ".repeat(40));
18590 let state = state_with_summary(&line, "- nothing");
18591 let m = pr_message(&state, 'A');
18592 assert!(m.title.starts_with("title words"), "{}", m.title);
18593 assert_eq!(
18594 land::merge_subject(&m.title, &landing_subject_source(&state)),
18595 m.title
18596 );
18597 assert_eq!(
18599 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18600 "feat: renamed by hand"
18601 );
18602 }
18603
18604 #[tokio::test]
18605 async fn github_text_rewrite_is_bounded_and_falls_back() {
18606 crate::run::pin_test_home();
18607 for (reply, accepted) in [
18608 (
18609 "TITLE: fix: retries\nAdd retries for failed requests.",
18610 true,
18611 ),
18612 (
18613 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18614 false,
18615 ),
18616 (
18617 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18618 true,
18619 ),
18620 (
18621 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18622 true,
18623 ),
18624 (
18625 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18626 true,
18627 ),
18628 (
18629 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18630 false,
18631 ),
18632 ("TITLE: fix: retries\n", false),
18633 ] {
18634 let dir = tempfile::tempdir().unwrap();
18635 let mut runner = runner_at(RunStatus::Gating);
18636 runner.state = state_with_summary(
18637 "add retries",
18638 "TITLE: fix: retries\n日本語の説明を書きます。",
18639 );
18640 runner.state.repo = dir.path().to_owned();
18641 runner.state.candidates[0].worktree = dir.path().to_owned();
18642 let mut author = spec("alpha");
18643 author.command = vec![
18644 "sh".into(),
18645 "-c".into(),
18646 "printf '%s' \"$REWRITE_REPLY\"".into(),
18647 ];
18648 author.env.insert("REWRITE_REPLY".into(), reply.into());
18649 runner.state.config.agents = vec![author];
18650 seed_public_summary(&mut runner);
18651 let winner = runner.state.candidates[0].clone();
18652 let first = runner
18653 .guarded_pr_message(&winner, None, true)
18654 .await
18655 .unwrap();
18656 let message = if accepted {
18657 first.expect("a passing rewrite never asks")
18658 } else {
18659 assert!(first.is_none());
18661 assert!(runner.state.parked);
18662 let store = ask::Questions::open();
18663 let asked: Vec<_> = store
18664 .list()
18665 .into_iter()
18666 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18667 .collect();
18668 assert_eq!(asked.len(), 1);
18669 assert!(!asked[0].detail.contains("secret"));
18670 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18671 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18672 assert_eq!(
18673 asked[0]
18674 .choices
18675 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18676 rewrite.is_some()
18677 );
18678 store
18681 .update(&asked[0].id, |q| {
18682 q.abandon("test");
18683 Ok(())
18684 })
18685 .unwrap();
18686 let again = runner
18687 .guarded_pr_message(&winner, None, true)
18688 .await
18689 .unwrap()
18690 .expect("an abandoned question falls back");
18691 let n = store
18692 .list()
18693 .into_iter()
18694 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18695 .count();
18696 assert_eq!(n, 1, "asked once per rejected text");
18697 again
18698 };
18699 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18700 assert!(!message.body.contains("secret"));
18701 assert!(!message.body.contains("prefix"));
18702 assert!(!message.title.contains("secret"));
18703 assert_eq!(
18704 message.body.contains("Add retries for failed requests."),
18705 accepted
18706 );
18707 assert_eq!(
18708 runner.state.seats["github-text-rewrite"].turns, 1,
18709 "only one rewrite invocation"
18710 );
18711 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18712 if accepted {
18713 assert!(runner.state.github_text.is_none());
18714 assert!(!runner.state.parked);
18715 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18716 assert!(message.body.contains("## Final verification"));
18717 assert!(message.body.contains("Evidence: final diff"));
18718 assert!(
18719 message
18720 .body
18721 .contains(&format!("magi:run/{}", runner.state.id))
18722 );
18723 }
18724 if !accepted {
18725 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18726 assert!(
18727 message
18728 .body
18729 .contains(&format!("magi:run/{}", runner.state.id))
18730 );
18731 }
18732 }
18733 }
18734
18735 fn scripted(id: &str, reply: &str) -> AgentSpec {
18736 let mut a = spec(id);
18737 a.command = vec![
18738 "sh".into(),
18739 "-c".into(),
18740 "printf '%s' \"$REWRITE_REPLY\"".into(),
18741 ];
18742 a.env.insert("REWRITE_REPLY".into(), reply.into());
18743 a
18744 }
18745
18746 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18747 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18748
18749 fn seed_public_summary(runner: &mut Runner) {
18751 let repo = &runner.state.repo;
18752 let g = |args: &[&str]| {
18753 let out = std::process::Command::new("git")
18754 .args(args)
18755 .current_dir(repo)
18756 .quiet()
18757 .output()
18758 .unwrap();
18759 assert!(out.status.success(), "git {args:?}");
18760 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18761 };
18762 g(&["init", "-q", "-b", "main"]);
18763 g(&[
18764 "-c",
18765 "user.name=Test",
18766 "-c",
18767 "user.email=test@example.invalid",
18768 "commit",
18769 "-q",
18770 "--allow-empty",
18771 "-m",
18772 "feat: fixture",
18773 ]);
18774 let head = g(&["rev-parse", "HEAD"]);
18775 g(&["update-ref", "refs/remotes/origin/main", &head]);
18776 let evidence = crate::pr_outcome::summary_evidence(
18777 &runner.state,
18778 crate::pr_outcome::FinalHead::At(&head),
18779 );
18780 let generated = &runner.state.candidates[0].summary;
18781 runner.state.change_summary = Some(crate::run::ChangeSummary {
18782 base: head.clone(),
18783 head,
18784 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18785 title: summary_title(generated).unwrap(),
18786 body: summary_without_title(generated),
18787 });
18788 }
18789
18790 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18791 let mut runner = runner_at(RunStatus::Gating);
18792 runner.state = state_with_summary("add retries", JA_SUMMARY);
18793 if review_only {
18794 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18795 }
18796 runner.state.repo = dir.to_owned();
18797 runner.state.candidates[0].worktree = dir.to_owned();
18798 runner.state.config.agents = agents;
18799 seed_public_summary(&mut runner);
18800 runner
18801 }
18802
18803 fn gate_questions(runner: &Runner) -> usize {
18804 ask::Questions::open()
18805 .list()
18806 .into_iter()
18807 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18808 .count()
18809 }
18810
18811 #[tokio::test]
18812 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18813 crate::run::pin_test_home();
18814 let dir = tempfile::tempdir().unwrap();
18815 let mut runner = rewrite_runner(
18816 dir.path(),
18817 true,
18818 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18819 );
18820 runner.state.config.roles.language_judge =
18821 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18822 let winner = runner.state.candidates[0].clone();
18823 let msg = runner
18824 .guarded_pr_message(&winner, None, true)
18825 .await
18826 .unwrap()
18827 .expect("an English rewrite posts without asking");
18828 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18829 assert!(msg.body.contains("<details>kept</details>"));
18830 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18831 assert_eq!(gate_questions(&runner), 0);
18832 assert!(runner.state.github_text.is_none());
18833 assert!(!runner.state.seats.contains_key("impl-A"));
18834 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18835 }
18836
18837 #[tokio::test]
18838 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18839 crate::run::pin_test_home();
18840 let dir = tempfile::tempdir().unwrap();
18841 let mut runner = rewrite_runner(
18842 dir.path(),
18843 false,
18844 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18845 );
18846 runner.state.config.roles.language_judge =
18847 Some(crate::config::AgentChoice::One("beta".into()));
18848 runner.state.seats.insert(
18849 "impl-A".into(),
18850 SeatState::new("impl-A", "alpha", runner.state.seed),
18851 );
18852 let winner = runner.state.candidates[0].clone();
18853 let msg = runner
18854 .guarded_pr_message(&winner, None, true)
18855 .await
18856 .unwrap()
18857 .expect("rewritten");
18858 assert!(msg.body.contains("<details>kept</details>"));
18859 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18860 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18861 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18862 }
18863
18864 #[tokio::test]
18865 async fn rewrite_reuses_a_resumable_implementer_session() {
18866 crate::run::pin_test_home();
18867 let dir = tempfile::tempdir().unwrap();
18868 let mut runner = rewrite_runner(
18869 dir.path(),
18870 false,
18871 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18872 );
18873 runner.state.config.roles.language_judge =
18874 Some(crate::config::AgentChoice::One("beta".into()));
18875 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18876 seat.turns = 1;
18877 runner.state.seats.insert("impl-A".into(), seat);
18878 let winner = runner.state.candidates[0].clone();
18879 runner
18880 .guarded_pr_message(&winner, None, true)
18881 .await
18882 .unwrap()
18883 .expect("rewritten");
18884 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18885 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18886 }
18887
18888 #[tokio::test]
18889 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
18890 crate::run::pin_test_home();
18893 let dir = tempfile::tempdir().unwrap();
18894 let mut exhausted = spec("alpha");
18895 exhausted.command = vec!["false".into()];
18896 let mut runner = rewrite_runner(
18897 dir.path(),
18898 false,
18899 vec![exhausted, scripted("beta", EN_REWRITE)],
18900 );
18901 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18903 "alpha".into(),
18904 "beta".into(),
18905 ]));
18906 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18907 seat.turns = 1;
18908 runner.state.seats.insert("impl-A".into(), seat);
18909 let winner = runner.state.candidates[0].clone();
18910 let msg = runner
18911 .guarded_pr_message(&winner, None, true)
18912 .await
18913 .unwrap()
18914 .expect("the role rewrites after the implementer fails");
18915 assert!(msg.body.contains("`src/net.rs`"));
18916 assert_eq!(
18917 runner.state.seats["impl-A"].turns, 2,
18918 "the implementer was tried once"
18919 );
18920 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18921 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18922 }
18923
18924 #[tokio::test]
18925 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18926 crate::run::pin_test_home();
18927 let dir = tempfile::tempdir().unwrap();
18928 let mut broken = spec("bad");
18929 broken.command = vec!["false".into()];
18930 let mut runner = rewrite_runner(
18931 dir.path(),
18932 true,
18933 vec![
18934 broken,
18935 scripted("noformat", "Add retries."),
18936 scripted("good", EN_REWRITE),
18937 ],
18938 );
18939 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18940 "bad".into(),
18941 "noformat".into(),
18942 "good".into(),
18943 ]));
18944 let winner = runner.state.candidates[0].clone();
18945 let msg = runner
18946 .guarded_pr_message(&winner, None, true)
18947 .await
18948 .unwrap()
18949 .expect("the third agent rewrites");
18950 assert!(msg.body.contains("`src/net.rs`"));
18951 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18952 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18953 }
18954
18955 #[tokio::test]
18956 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18957 crate::run::pin_test_home();
18958 for (agents, role, saved) in [
18960 (
18961 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18962 Some("beta"),
18963 true,
18964 ),
18965 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18966 (Vec::new(), None, false),
18967 ] {
18968 let dir = tempfile::tempdir().unwrap();
18969 let mut runner = rewrite_runner(dir.path(), true, agents);
18970 runner.state.config.roles.language_judge =
18971 role.map(|r| crate::config::AgentChoice::One(r.into()));
18972 let winner = runner.state.candidates[0].clone();
18973 let first = runner
18974 .guarded_pr_message(&winner, None, true)
18975 .await
18976 .unwrap();
18977 assert!(first.is_none(), "withheld text parks the run");
18978 assert!(runner.state.parked);
18979 assert_eq!(gate_questions(&runner), 1);
18980 let g = runner.state.github_text.as_ref().unwrap();
18981 assert_eq!(g.rewrite.is_some(), saved);
18982 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18983 }
18984 }
18985
18986 #[tokio::test]
18987 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18988 crate::run::pin_test_home();
18989 let dir = tempfile::tempdir().unwrap();
18990 let mut runner = runner_at(RunStatus::Gating);
18991 runner.state =
18992 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18993 runner.state.repo = dir.path().to_owned();
18994 runner.state.candidates[0].worktree = dir.path().to_owned();
18995 let mut author = spec("alpha");
18996 author.command = vec![
18997 "sh".into(),
18998 "-c".into(),
18999 "printf '%s' \"$REWRITE_REPLY\"".into(),
19000 ];
19001 author.env.insert(
19002 "REWRITE_REPLY".into(),
19003 format!(
19004 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19005 "English sentence. ".repeat(600)
19006 ),
19007 );
19008 runner.state.config.agents = vec![author];
19009 seed_public_summary(&mut runner);
19010 let winner = runner.state.candidates[0].clone();
19011 assert!(
19012 runner
19013 .guarded_pr_message(&winner, None, true)
19014 .await
19015 .unwrap()
19016 .is_none()
19017 );
19018 let record = runner.state.github_text.clone().unwrap();
19019 let rewrite = record.rewrite.clone().unwrap();
19020 assert_eq!(rewrite.failed, ["body-language"]);
19021 assert!(
19022 rewrite
19023 .body
19024 .ends_with(&format!("magi:run/{}", runner.state.id))
19025 );
19026 let store = ask::Questions::open();
19027 let q = store.get(record.question.as_ref().unwrap()).unwrap();
19028 assert!(
19029 q.choices
19030 .contains(&crate::github_text::USE_REWRITE.to_owned())
19031 );
19032 for label in [
19033 "ORIGINAL",
19034 "FALLBACK",
19035 "REWRITE",
19036 "Rules still failed: body-language",
19037 "truncated",
19038 ] {
19039 assert!(q.detail.contains(label), "{label}");
19040 }
19041 let artifact = std::fs::read_to_string(crate::run::artifact_path(
19042 &runner.state,
19043 &crate::github_text::artifact_name(&record.fingerprint),
19044 ))
19045 .unwrap();
19046 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19047 runner.state =
19049 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19050 store
19051 .update(&q.id, |q| {
19052 q.say("まだ日本語")?;
19053 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19054 })
19055 .unwrap();
19056 assert!(
19057 runner
19058 .guarded_pr_message(&winner, None, true)
19059 .await
19060 .unwrap()
19061 .is_none()
19062 );
19063 let retry = runner.state.github_text.as_ref().unwrap();
19064 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19065 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19066 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19067 assert!(
19068 q.posting_comparison
19069 .as_ref()
19070 .unwrap()
19071 .matches_detail(&q.detail)
19072 );
19073 store
19074 .update(&q.id, |q| {
19075 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19076 })
19077 .unwrap();
19078 let message = runner
19079 .guarded_pr_message(&winner, None, true)
19080 .await
19081 .unwrap()
19082 .unwrap();
19083 assert_eq!(message.title, rewrite.title);
19084 assert_eq!(message.body, rewrite.body);
19085 assert_eq!(
19086 runner.state.github_text.as_ref().unwrap().choice,
19087 Some(GithubTextChoice::Rewrite)
19088 );
19089 let again = runner
19090 .guarded_pr_message(&winner, None, true)
19091 .await
19092 .unwrap()
19093 .unwrap();
19094 assert_eq!(again.title, message.title);
19095 assert_eq!(again.body, message.body);
19096 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19097 assert!(
19098 !runner
19099 .state
19100 .events
19101 .iter()
19102 .any(|e| e.message.contains(&rewrite.title)
19103 || e.message.contains("English sentence."))
19104 );
19105 }
19106
19107 #[tokio::test]
19108 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19109 crate::run::pin_test_home();
19110 let dir = tempfile::tempdir().unwrap();
19111 let mut runner = runner_at(RunStatus::Gating);
19112 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19113 runner.state.repo = dir.path().to_owned();
19114 runner.state.candidates[0].worktree = dir.path().to_owned();
19115 seed_public_summary(&mut runner);
19116 let winner = runner.state.candidates[0].clone();
19117 assert!(
19118 runner
19119 .guarded_pr_message(&winner, None, true)
19120 .await
19121 .unwrap()
19122 .is_none()
19123 );
19124 let record = runner.state.github_text.as_ref().unwrap();
19125 assert!(record.rewrite.is_none());
19126 let mut old = serde_json::to_value(record).unwrap();
19128 old.as_object_mut().unwrap().remove("rewrite");
19129 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19130 assert!(old.rewrite.is_none());
19131 let id = record.question.clone().unwrap();
19132 ask::Questions::open()
19133 .update(&id, |q| {
19134 q.choices.push(crate::github_text::USE_REWRITE.into());
19135 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19136 })
19137 .unwrap();
19138 let message = runner
19139 .guarded_pr_message(&winner, None, true)
19140 .await
19141 .unwrap()
19142 .unwrap();
19143 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19144 assert_eq!(
19145 runner.state.github_text.as_ref().unwrap().choice,
19146 Some(GithubTextChoice::Fallback)
19147 );
19148 }
19149
19150 #[tokio::test]
19151 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19152 crate::run::pin_test_home();
19153 let dir = tempfile::tempdir().unwrap();
19154 let mut runner = runner_at(RunStatus::Gating);
19155 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19156 runner.state.repo = dir.path().to_owned();
19157 runner.state.candidates[0].worktree = dir.path().to_owned();
19158 seed_public_summary(&mut runner);
19159 let winner = runner.state.candidates[0].clone();
19160 assert!(
19161 runner
19162 .guarded_pr_message(&winner, None, true)
19163 .await
19164 .unwrap()
19165 .is_none()
19166 );
19167 let store = ask::Questions::open();
19168 let find = |run: &str| {
19169 store
19170 .list()
19171 .into_iter()
19172 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19173 .unwrap()
19174 };
19175 let run = runner.state.id.clone();
19176 let q = find(&run);
19177 let fp = runner
19178 .state
19179 .github_text
19180 .as_ref()
19181 .unwrap()
19182 .fingerprint
19183 .clone();
19184 let name = crate::github_text::artifact_name(&fp);
19185 let path = crate::run::artifact_path(&runner.state, &name);
19186 assert!(q.detail.contains(&path.display().to_string()));
19187 assert!(q.detail.contains("title-language: title"));
19188 let saved = std::fs::read_to_string(&path).expect("artifact written");
19189 assert!(saved.contains("日本語のタイトル"));
19190 assert!(q.cwd.is_none());
19191 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19192 let say = |id: &str, text: &str, choice: &str| {
19194 store
19195 .update(id, |q| {
19196 q.thread.push(ask::Turn {
19197 who: ask::Who::Operator,
19198 body: text.to_owned(),
19199 at: jiff::Timestamp::now(),
19200 note: None,
19201 });
19202 q.answer(ask::Answer::Choice(choice.to_owned()))
19203 })
19204 .unwrap();
19205 };
19206 say(&q.id, "まだ日本語", "use my text");
19207 assert!(
19208 runner
19209 .guarded_pr_message(&winner, None, true)
19210 .await
19211 .unwrap()
19212 .is_none()
19213 );
19214 let second = find(&run);
19215 assert_ne!(second.id, q.id);
19216 assert!(second.detail.contains("did not pass"));
19217 assert!(second.detail.contains("日本語のタイトル"));
19220 assert!(!second.detail.contains("まだ日本語"));
19221 assert!(second.detail.contains(&path.display().to_string()));
19222 say(&second.id, "fix: retry failed requests", "use my text");
19223 let message = runner
19224 .guarded_pr_message(&winner, None, true)
19225 .await
19226 .unwrap()
19227 .expect("a vetted replacement is posted");
19228 assert_eq!(message.title, "fix: retry failed requests");
19229 assert_eq!(
19230 runner
19231 .state
19232 .github_text
19233 .as_ref()
19234 .unwrap()
19235 .chosen_title
19236 .as_deref(),
19237 Some("fix: retry failed requests")
19238 );
19239 assert!(runner.state.github_text.as_ref().unwrap().resolved);
19240 let again = runner
19242 .guarded_pr_message(&winner, None, true)
19243 .await
19244 .unwrap()
19245 .unwrap();
19246 assert_eq!(again.title, message.title);
19247 assert_eq!(
19248 store
19249 .list()
19250 .into_iter()
19251 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19252 .count(),
19253 2
19254 );
19255 }
19256
19257 #[test]
19258 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19259 let mut state = state_with_summary(
19263 "add retries",
19264 "TITLE: fix(web): batch reads\n- reads run.json once",
19265 );
19266 state.config.graph.language = "ja".to_owned();
19267 let m = pr_message(&state, 'A');
19268 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19269 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19270
19271 let task = "今回やってほしいこと: results projector を直す";
19274 let mut state = state_with_summary(task, "- no title line");
19275 state.config.graph.language = "ja".to_owned();
19276 let m = pr_message(&state, 'A');
19277 assert_eq!(
19278 m.title,
19279 format!("chore: land candidate A of run {}", state.id)
19280 );
19281 assert!(
19282 m.body.contains(&format!(
19283 "<summary>Original task</summary>\n\n{task}\n\n</details>"
19284 )),
19285 "{}",
19286 m.body
19287 );
19288 }
19289
19290 #[test]
19291 fn pr_message_scrubs_home_paths_and_addresses() {
19292 let state = state_with_summary(
19293 "fix it in /Users/someone/src/x",
19294 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19295 );
19296 let m = pr_message(&state, 'A');
19297 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19298 assert!(!m.body.contains(leak), "{}", m.body);
19299 }
19300 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19301 }
19302
19303 #[test]
19304 fn pr_message_survives_a_task_that_closes_details() {
19305 let state = state_with_summary("a </details> b", "TITLE: fix: x");
19306 let m = pr_message(&state, 'A');
19307 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19308 }
19309
19310 #[test]
19311 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19312 let cmd = manual_merge_command(
19313 MergeStyle::Squash,
19314 Path::new("/repo"),
19315 "b",
19316 "fix: \"quoted\" $(x) `y`\n\nbody",
19317 );
19318 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19319 }
19320
19321 #[test]
19322 fn manual_merge_command_matches_the_configured_style() {
19323 let repo = Path::new("/repo");
19324 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19325
19326 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19327 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19328
19329 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19330 assert_eq!(
19331 squash,
19332 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19333 \"Merge magi run 0832 (candidate A)\""
19334 );
19335
19336 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19337 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19338 }
19339
19340 #[test]
19341 fn a_nudge_gets_a_quarter_of_the_budget() {
19342 assert_eq!(retry_budget(secs(1200), true), secs(300));
19344 assert_eq!(retry_budget(secs(3600), true), secs(900));
19345 }
19346
19347 #[test]
19348 fn a_resent_prompt_keeps_the_whole_budget() {
19349 assert_eq!(retry_budget(secs(1200), false), secs(1200));
19352 assert_eq!(retry_budget(secs(60), false), secs(60));
19353 }
19354
19355 #[test]
19356 fn the_floor_never_exceeds_the_original_budget() {
19357 assert_eq!(retry_budget(secs(60), true), secs(60));
19361 assert_eq!(retry_budget(secs(480), true), secs(120));
19362 assert_eq!(retry_budget(secs(0), true), secs(0));
19363 }
19364
19365 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19366 agent::CommandEvidence {
19367 id: "item1".to_owned(),
19368 description: "cargo test".to_owned(),
19369 exit_code,
19370 result_summary: String::new(),
19371 source: "codex".to_owned(),
19372 }
19373 }
19374
19375 #[test]
19376 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19377 assert!(!has_unconfirmed_command(&[]));
19381 }
19382
19383 #[test]
19384 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19385 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19389 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19390 assert!(!has_unconfirmed_command(&[
19391 evidence(Some(0)),
19392 evidence(Some(101))
19393 ]));
19394 }
19395
19396 #[test]
19397 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19398 assert!(has_unconfirmed_command(&[
19399 evidence(Some(0)),
19400 evidence(None)
19401 ]));
19402 }
19403
19404 #[test]
19405 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19406 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19407 assert_eq!(
19408 verified_noop_claim(true, &[], text).as_deref(),
19409 Some("already fixed by b32cfc4, on main.")
19410 );
19411 }
19412
19413 #[test]
19414 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19415 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19418 assert!(verified_noop_claim(false, &[], text).is_none());
19419 }
19420
19421 #[test]
19422 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19423 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19424 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19425 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19427 }
19428
19429 #[test]
19430 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19431 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19432 }
19433
19434 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19437 runner.state.candidates = shape
19438 .iter()
19439 .enumerate()
19440 .map(|(i, &(empty, verified))| Candidate {
19441 index: i,
19442 label: (b'A' + i as u8) as char,
19443 agent: "sonnet".to_owned(),
19444 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19445 worktree: PathBuf::from(format!("/wt/{i}")),
19446 summary: String::new(),
19447 stat: String::new(),
19448 files: 0,
19449 commits: 0,
19450 empty,
19451 failed: None,
19452 verified_noop: verified.map(str::to_owned),
19453 duration_ms: 0,
19454 folded: false,
19455 })
19456 .collect();
19457 }
19458
19459 #[test]
19460 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19461 ask_test_home();
19462 let mut runner = runner_at(RunStatus::Implementing);
19463 set_candidates(
19464 &mut runner,
19465 &[
19466 (true, Some("already on main at b32cfc4")),
19467 (true, Some("same fix, see the existing test")),
19468 ],
19469 );
19470
19471 runner
19472 .after_implement()
19473 .expect("a verified no-op is not an error");
19474
19475 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19476 }
19477
19478 #[test]
19479 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19480 ask_test_home();
19481 let mut runner = runner_at(RunStatus::Implementing);
19482 set_candidates(
19486 &mut runner,
19487 &[(true, Some("already on main at b32cfc4")), (true, None)],
19488 );
19489
19490 let err = runner
19491 .after_implement()
19492 .expect_err("an unverified empty candidate must still fail the run");
19493
19494 assert!(
19495 err.to_string().contains("no candidate produced a change"),
19496 "{err}"
19497 );
19498 assert_eq!(runner.state.status, RunStatus::Failed);
19499 }
19500
19501 #[test]
19502 fn after_implement_still_fails_an_ordinary_all_empty_run() {
19503 ask_test_home();
19504 let mut runner = runner_at(RunStatus::Implementing);
19505 set_candidates(&mut runner, &[(true, None), (true, None)]);
19506
19507 let err = runner
19508 .after_implement()
19509 .expect_err("no candidate declared anything; this is an ordinary failure");
19510
19511 assert!(
19512 err.to_string().contains("no candidate produced a change"),
19513 "{err}"
19514 );
19515 assert_eq!(runner.state.status, RunStatus::Failed);
19516 }
19517}
19518
19519#[derive(Debug)]
19522struct GraftRestoreFailed(String);
19523
19524impl std::fmt::Display for GraftRestoreFailed {
19525 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19526 write!(
19527 f,
19528 "could not restore the winner to its pre-graft commit: {}",
19529 self.0
19530 )
19531 }
19532}
19533
19534impl std::error::Error for GraftRestoreFailed {}
19535
19536#[cfg(test)]
19540mod resource_wait_tests {
19541 use super::*;
19542 use std::collections::BTreeMap;
19543
19544 fn sh(id: &str, script: &str) -> AgentSpec {
19545 AgentSpec {
19546 id: id.to_owned(),
19547 kind: crate::config::AgentKind::Command,
19548 model: None,
19549 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19550 extra_args: Vec::new(),
19551 env: BTreeMap::new(),
19552 prompt_delivery: None,
19553 max_turn_secs: None,
19554 }
19555 }
19556
19557 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19558 SeatJob {
19559 seat: SeatState::new("impl-A", &spec.id, 1),
19560 spec,
19561 cwd: dir.to_path_buf(),
19562 prompt: "do it".to_owned(),
19563 timeout,
19564 allow_write: true,
19565 sessions: false,
19566 artifacts: dir.join("art"),
19567 stem: "impl-A".to_owned(),
19568 handover: None,
19569 }
19570 }
19571
19572 fn state(resource_wait_secs: u64) -> RunState {
19573 let mut config = Config::default();
19574 config.graph.timeout_resource_wait = resource_wait_secs;
19575 RunState::new(
19576 PathBuf::from("/repo"),
19577 "main".to_owned(),
19578 "abc1234".to_owned(),
19579 "task".to_owned(),
19580 config,
19581 )
19582 }
19583
19584 fn hold(cache: &Path) -> crate::cache::Guard {
19585 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19586 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19587 crate::cache::AcquireOutcome::Acquired(g) => g,
19588 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19589 }
19590 }
19591
19592 async fn run(
19593 jobs: Vec<SeatJob>,
19594 sem: Arc<Semaphore>,
19595 cache: Option<&Path>,
19596 st: &mut RunState,
19597 ) -> Vec<(usize, SeatState, AgentOutcome)> {
19598 let prompts = Prompts::default();
19599 let ctx = WaveCtx {
19600 run: "r",
19601 node: "implement",
19602 prompts: &prompts,
19603 cache,
19604 round: None,
19605 carry_seats: false,
19606 pauses: None,
19607 };
19608 wave(jobs, sem, &ctx, st, 0).await
19609 }
19610
19611 #[tokio::test]
19612 async fn verification_cache_wait_preserves_the_command_budget() {
19613 crate::run::pin_test_home();
19614 let tmp = tempfile::tempdir().unwrap();
19615 let cache = tmp.path().join("target");
19616 let guard = hold(&cache);
19617 let release = tokio::spawn(async move {
19618 tokio::time::sleep(Duration::from_millis(100)).await;
19619 guard.release();
19620 });
19621 let mut st = state(10);
19622 let budget = Duration::from_millis(50);
19623 let wait = repository_wait(&st, None);
19624 let (outcomes, _) = with_cache_lease(
19625 &mut st,
19626 Some(&cache),
19627 "gate",
19628 "gate",
19629 tmp.path(),
19630 "abc1234",
19631 budget,
19632 "test gate",
19633 &wait,
19634 |_, available| async move {
19635 assert_eq!(
19636 available, budget,
19637 "contention must not shorten verification"
19638 );
19639 (Vec::new(), false, Vec::new())
19640 },
19641 )
19642 .await
19643 .unwrap();
19644 release.await.unwrap();
19645 assert!(outcomes.is_empty(), "{outcomes:?}");
19646 assert!(
19647 st.events
19648 .iter()
19649 .any(|event| event.message.contains("waiting for"))
19650 );
19651 assert!(st.resource_wait.is_none());
19652 hold(&cache).release();
19654 }
19655
19656 #[tokio::test]
19657 async fn verification_cache_wait_expires_without_starting_commands() {
19658 crate::run::pin_test_home();
19659 let tmp = tempfile::tempdir().unwrap();
19660 let cache = tmp.path().join("target");
19661 let _held = hold(&cache);
19662 let mut st = state(0);
19663 let wait = repository_wait(&st, None);
19664 let (outcomes, _) = with_cache_lease(
19665 &mut st,
19666 Some(&cache),
19667 "gate",
19668 "gate",
19669 tmp.path(),
19670 "abc1234",
19671 Duration::from_secs(60),
19672 "test gate",
19673 &wait,
19674 |_, _| async { panic!("commands must not start without the cache lease") },
19675 )
19676 .await
19677 .unwrap();
19678 assert_eq!(outcomes.len(), 1);
19679 assert!(outcomes[0].resource_blocked);
19680 assert!(st.resource_wait.is_none());
19681 assert!(matches!(
19682 st.resource_block.unwrap().cause,
19683 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19684 ));
19685 }
19686
19687 #[tokio::test]
19688 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19689 crate::run::pin_test_home();
19690 let tmp = tempfile::tempdir().unwrap();
19691 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19692 let cache = tmp.path().join("cache");
19693 let mut state = state(21600);
19694 let wait = repository_wait(&state, None);
19695 let (outcomes, _) = with_cache_lease(
19696 &mut state,
19697 Some(&cache),
19698 "gate",
19699 "gate",
19700 tmp.path(),
19701 "head",
19702 Duration::from_secs(60),
19703 "gate",
19704 &wait,
19705 |_, _| async { panic!("freshness failure must not start verification") },
19706 )
19707 .await
19708 .unwrap();
19709 assert_eq!(outcomes.len(), 1);
19710 assert!(outcomes[0].resource_blocked);
19711 let block = state.resource_block.unwrap();
19712 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19713 assert!(!block.description().contains("timed out"));
19714 }
19715
19716 #[tokio::test]
19717 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19718 crate::run::pin_test_home();
19719 let tmp = tempfile::tempdir().unwrap();
19720 let cache = tmp.path().join("cache");
19721 let _held = hold(&cache);
19722 let mut state = state(21600);
19723 let pause = Pause::new();
19724 let interrupt = Pause::new();
19725 let cause = ParkCause::Operator;
19726 pause.park_for(cause.clone());
19727 let prompts = Prompts::default();
19728 let ctx = WaveCtx {
19729 run: "test",
19730 node: "fix",
19731 prompts: &prompts,
19732 cache: Some(&cache),
19733 round: None,
19734 carry_seats: false,
19735 pauses: Some((&pause, &interrupt)),
19736 };
19737 let result = wave(
19738 vec![job(
19739 sh("writer", "exit 99"),
19740 tmp.path(),
19741 Duration::from_secs(1),
19742 )],
19743 Arc::new(Semaphore::new(1)),
19744 &ctx,
19745 &mut state,
19746 0,
19747 )
19748 .await;
19749 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19750 assert_eq!(result[0].1.turns, 0);
19751 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19752 assert_eq!(
19753 state.resource_block.unwrap().cause,
19754 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19755 );
19756 assert!(state.active.is_empty());
19757 }
19758
19759 #[tokio::test]
19760 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19761 crate::run::pin_test_home();
19762 let tmp = tempfile::tempdir().unwrap();
19763 let cache = tmp.path().join("target-a");
19764 let guard = hold(&cache);
19765 tokio::spawn(async move {
19767 tokio::time::sleep(Duration::from_millis(1500)).await;
19768 guard.release();
19769 });
19770 let mut st = state(60);
19771 let results = run(
19772 vec![job(
19773 sh("alpha", "echo ok"),
19774 tmp.path(),
19775 Duration::from_secs(1),
19776 )],
19777 Arc::new(Semaphore::new(1)),
19778 Some(&cache),
19779 &mut st,
19780 )
19781 .await;
19782 assert!(
19783 matches!(results[0].2, AgentOutcome::Ok(_)),
19784 "the wait must not be charged to the agent"
19785 );
19786 assert!(st.resource_block.is_none());
19787 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19788 assert!(st.resource_wait.is_none() && st.active.is_empty());
19789 }
19790
19791 #[tokio::test]
19792 #[cfg(unix)]
19793 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19794 use std::os::fd::AsRawFd as _;
19795 crate::run::pin_test_home();
19796 let tmp = tempfile::tempdir().unwrap();
19797 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19798 let common = tmp.path().join(".git");
19799 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19800 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19801 assert_eq!(
19802 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19803 0
19804 );
19805 let mut st = state(1);
19806 st.config.coordination = crate::config::test_coordination();
19807 let mut reader = job(
19808 sh("reader", "echo must-not-run"),
19809 tmp.path(),
19810 Duration::from_secs(30),
19811 );
19812 reader.allow_write = false;
19813 let results = run(
19814 vec![reader.clone()],
19815 Arc::new(Semaphore::new(1)),
19816 None,
19817 &mut st,
19818 )
19819 .await;
19820 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19821 assert_eq!(results[0].1.turns, 0);
19822 assert!(FailClass::of(&results[0].2).is_none());
19823 assert_eq!(
19824 st.resource_block.as_ref().unwrap().kind,
19825 "repository coordination"
19826 );
19827 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19828 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19829 drop(external);
19830 let editor = crate::repo_lock::SeatLease::for_checkout(
19831 tmp.path(),
19832 true,
19833 crate::config::test_coordination().as_ref(),
19834 )
19835 .await
19836 .unwrap();
19837 let pause = Pause::new();
19838 let interrupt = Pause::new();
19839 let prompts = Prompts::default();
19840 let ctx = WaveCtx {
19841 run: "r",
19842 node: "review",
19843 prompts: &prompts,
19844 cache: None,
19845 round: None,
19846 carry_seats: false,
19847 pauses: Some((&pause, &interrupt)),
19848 };
19849 let (results, ()) = tokio::join!(
19850 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19851 async {
19852 tokio::time::sleep(Duration::from_millis(50)).await;
19853 pause.park();
19854 }
19855 );
19856 assert!(
19857 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19858 );
19859 assert_eq!(results[0].1.turns, 0);
19860 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19861 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19862 drop(editor);
19863 }
19864
19865 #[tokio::test]
19866 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19867 crate::run::pin_test_home();
19868 let tmp = tempfile::tempdir().unwrap();
19869 let cache = tmp.path().join("target-b");
19870 let _held = hold(&cache);
19871 let mut st = state(1);
19872 let results = run(
19873 vec![job(
19874 sh("alpha", "echo ok"),
19875 tmp.path(),
19876 Duration::from_secs(1),
19877 )],
19878 Arc::new(Semaphore::new(1)),
19879 Some(&cache),
19880 &mut st,
19881 )
19882 .await;
19883 let (_, seat, out) = &results[0];
19884 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19885 assert!(wave_blocked(&results));
19886 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19887 assert_eq!(seat.turns, 0, "nothing was asked");
19888 assert!(
19889 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19890 "no agent was spawned"
19891 );
19892 assert!(FailClass::of(out).is_none(), "not a failure class");
19893 let block = st.resource_block.as_ref().expect("recorded for the resume");
19894 assert_eq!(block.kind, "shared build cache");
19895 assert!(matches!(
19896 block.cause,
19897 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
19898 ));
19899 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19900 assert!(st.active.is_empty());
19901 }
19902
19903 #[tokio::test]
19904 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19905 crate::run::pin_test_home();
19906 let tmp = tempfile::tempdir().unwrap();
19907 let mut st = state(1);
19908 let results = run(
19909 vec![job(
19910 sh("alpha", "echo ok"),
19911 tmp.path(),
19912 Duration::from_secs(1),
19913 )],
19914 Arc::new(Semaphore::new(0)),
19915 None,
19916 &mut st,
19917 )
19918 .await;
19919 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19920 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19921 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19922 }
19923
19924 #[tokio::test]
19925 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19926 crate::run::pin_test_home();
19927 let tmp = tempfile::tempdir().unwrap();
19928 let mut st = state(60);
19929 let results = run(
19930 vec![job(
19931 sh("alpha", "sleep 30"),
19932 tmp.path(),
19933 Duration::from_secs(1),
19934 )],
19935 Arc::new(Semaphore::new(1)),
19936 None,
19937 &mut st,
19938 )
19939 .await;
19940 let out = &results[0].2;
19941 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19942 let class = FailClass::of(out).expect("a failure");
19943 assert_eq!(class, FailClass::Timeout);
19944 assert!(should_hand_over(None, &class), "existing handover policy");
19945 assert!(st.resource_block.is_none());
19946 }
19947}