1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530enum FixStep {
532 Resource,
533 Recorded { report_lost: bool },
534}
535
536struct FixClaim {
553 path: PathBuf,
554}
555
556impl FixClaim {
557 fn acquire(dir: &Path) -> Result<Self> {
558 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559 let path = dir.join("fix.lock");
560 let retry = crate::lock_file::Retry::new(None);
561 match Self::create(&path, retry) {
562 Ok(claim) => Ok(claim),
563 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564 if Self::reclaim_if_dead(&path) {
565 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566 } else {
567 bail!(
568 "another repair or `magi fix` is already running for this run ({} exists)",
569 path.display()
570 )
571 }
572 }
573 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574 }
575 }
576
577 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578 let mut f = retry.create(path)?;
579 use std::io::Write as _;
580 writeln!(f, "{}", std::process::id())?;
582 Ok(Self {
583 path: path.to_owned(),
584 })
585 }
586
587 fn reclaim_if_dead(path: &Path) -> bool {
591 let dead = std::fs::read_to_string(path)
592 .ok()
593 .and_then(|body| body.trim().parse::<u32>().ok())
594 .is_some_and(|pid| !crate::proc::pid_alive(pid));
595 dead && std::fs::remove_file(path).is_ok()
596 }
597}
598
599impl Drop for FixClaim {
600 fn drop(&mut self) {
601 let _ = std::fs::remove_file(&self.path);
602 }
603}
604
605struct DriverExecution<'a> {
608 runner: &'a mut Runner,
609 driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613 fn drop(&mut self) {
614 self.driver.release_local(&mut self.runner.state);
615 }
616}
617
618impl Runner {
619 pub async fn start(
621 repo: &Path,
622 instruction: String,
623 config: Config,
624 origin: Origin,
625 ) -> Result<Self> {
626 Self::start_naming(repo, instruction, "", config, origin).await
627 }
628
629 pub async fn start_naming(
632 repo: &Path,
633 instruction: String,
634 _also_scan: &str,
635 config: Config,
636 origin: Origin,
637 ) -> Result<Self> {
638 let repo = git::toplevel(repo).await?;
639 let missing = agent::missing_programs(&config.agents);
640 if !missing.is_empty() {
641 bail!(
642 "these agent programs are not on PATH: {}. Fix the roster in \
643 magi.toml or install them.",
644 missing.join(", ")
645 );
646 }
647 let base_branch =
648 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649 .await?;
650 let fetched_base = resolve_base(
651 &repo,
652 &base_branch,
653 &config.merge.remote,
654 config.coordination.as_ref(),
655 )
656 .await?;
657 let base_commit = fetched_base.commit.clone();
658 let roles = config.resolve_roles()?;
659 let max_parallel = config.graph.max_parallel.max(1);
660 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661 state.fetched_base = Some(fetched_base);
665 state.origin = Some(origin);
666 state.reserve()?;
667 state.event("start", format!("run {} created", state.id));
668 state.save()?;
669 Ok(Self {
670 state,
671 roles,
672 sem: Arc::new(Semaphore::new(max_parallel)),
673 pause: Pause::new(),
674 interrupt: Pause::new(),
675 })
676 }
677
678 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692 Self::review_taking_over(repo, branch, config, None, origin).await
693 }
694
695 pub async fn review_taking_over(
702 repo: &Path,
703 branch: &str,
704 config: Config,
705 takeover: Option<crate::handover::Takeover>,
706 origin: Origin,
707 ) -> Result<Self> {
708 let repo = git::toplevel(repo).await?;
709 let missing = agent::missing_programs(&config.agents);
710 if !missing.is_empty() {
711 bail!(
712 "these agent programs are not on PATH: {}. Fix the roster in \
713 magi.toml or install them.",
714 missing.join(", ")
715 );
716 }
717 let base_branch =
718 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719 .await?;
720 if base_branch == branch {
721 bail!("`{branch}` is the base branch; there is nothing to review against");
722 }
723 let fetched_base = resolve_base(
724 &repo,
725 &base_branch,
726 &config.merge.remote,
727 config.coordination.as_ref(),
728 )
729 .await?;
730 let base_commit = fetched_base.commit.clone();
731
732 let roles = config.resolve_roles()?;
733 let max_parallel = config.graph.max_parallel.max(1);
734 let mut state = RunState::new(
735 repo.clone(),
736 base_branch,
737 base_commit.clone(),
738 String::new(),
739 config,
740 );
741 state.fetched_base = Some(fetched_base);
742 state.origin = Some(origin);
743 state.reserve()?;
744
745 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752 earlier: Vec::new(),
753 home: crate::run::home(),
754 choice: None,
755 task_instruction: None,
756 });
757 let wait = repository_wait(&state, None);
758 let coordination_config = state.config.coordination.clone();
759 let coordination =
760 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761 .await?;
762 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763 if let Some(released) = &released {
764 state.event(
765 "release",
766 format!(
767 "took `{branch}` over from run {}: its worktree was released: {}",
768 crate::run::short_of(&released.old_id),
769 released.audit
770 ),
771 );
772 }
773 if let Some(choice) = takeover.choice.as_ref()
777 && let Err(e) =
778 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779 .await
780 {
781 if let Some(released) = &released {
782 released.restore(&repo, branch).await;
783 }
784 return Err(e.context("applying the owner's answer about the diverged branch"));
785 }
786 for id in takeover.earlier.iter().rev() {
792 if let Ok(old) = RunState::load_under(id, &takeover.home)
793 && let Some(head) = old.pr_head.clone()
794 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795 {
796 state.pr_head = Some(head);
797 break;
798 }
799 }
800 drop(coordination);
801 let opened = Self::open_review(
802 &repo,
803 branch,
804 state,
805 roles,
806 max_parallel,
807 base_commit,
808 &takeover,
809 )
810 .await;
811 if opened.is_err()
812 && let Some(released) = &released
813 {
814 let _coordination = crate::repo_lock::SeatLease::operation_with(
815 &repo,
816 coordination_config.as_ref(),
817 &wait,
818 )
819 .await?;
820 released.restore(&repo, branch).await;
821 }
822 opened
823 }
824
825 pub async fn recover_pr(
827 repo: &Path,
828 task: &str,
829 selection: crate::pr_recovery::Selection,
830 mut config: Config,
831 ) -> Result<Self> {
832 selection.configure(&mut config)?;
833 selection.check_task(task, &selection.instruction)?;
834 let base_branch =
835 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836 .await?;
837 let fetched_base = resolve_base(
838 repo,
839 &base_branch,
840 &config.merge.remote,
841 config.coordination.as_ref(),
842 )
843 .await?;
844 let base_commit = fetched_base.commit.clone();
845 selection
846 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847 .await?;
848 Box::pin(selection.verify_source(repo)).await?;
849 if let Some(proof) = &selection.source_proof {
850 anyhow::ensure!(
851 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852 "source rebase base is outside the fetched landing base"
853 );
854 }
855 let roles = config.resolve_roles()?;
856 let max_parallel = config.graph.max_parallel.max(1);
857 let mut state = RunState::new(
858 repo.to_path_buf(),
859 base_branch,
860 base_commit.clone(),
861 selection.instruction.clone(),
862 config,
863 );
864 state.fetched_base = Some(fetched_base);
865 state.origin = Some(crate::run::Origin::queue(task));
866 state.pr_head = Some(selection.branch.clone());
867 state.pr_recovery = Some(selection.clone());
868 state.reserve()?;
869 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870 state.discard_reserved();
872 return Err(e);
873 }
874 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875 }
876
877 async fn open_selected(
878 repo: &Path,
879 state: RunState,
880 roles: ResolvedRoles,
881 max_parallel: usize,
882 base_commit: String,
883 selection: crate::pr_recovery::Selection,
884 ) -> Result<Self> {
885 let mut state = state;
886 let branch = state.branch_for('A');
887 let wait = repository_wait(&state, None);
888 let prepared = async {
889 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890 git::git(repo, &["branch", &branch, selection.source()]).await?;
891 drop(coordination);
892 anyhow::Ok(())
893 }
894 .await;
895 if let Err(e) = prepared {
896 state.discard_reserved();
899 return Err(e);
900 }
901 state.event(
902 "recovery",
903 format!(
904 "selected implementation {} for {} (public lease {})",
905 selection.source(),
906 selection.url,
907 selection.head
908 ),
909 );
910 let takeover = crate::handover::Takeover {
911 earlier: Vec::new(),
912 home: crate::run::home(),
913 choice: None,
914 task_instruction: Some(selection.instruction.clone()),
915 };
916 if let Err(e) = state.save() {
917 state.discard_reserved();
918 return Err(e);
919 }
920 let id = state.id.clone();
921 let opened = Self::open_review(
922 repo,
923 &branch,
924 state,
925 roles,
926 max_parallel,
927 base_commit,
928 &takeover,
929 )
930 .await;
931 match opened {
932 Ok(runner) => Ok(runner),
933 Err(e) => {
934 if let Ok(mut partial) = RunState::load(&id) {
935 partial.status = RunStatus::Blocked;
936 partial.driver_exited = true;
937 partial.event(
938 "recovery",
939 format!("startup failed; retained source and candidate {branch}: {e:#}"),
940 );
941 partial.save()?;
942 }
943 Err(crate::pr_recovery::StartupFailed {
944 run: id,
945 cause: format!("{e:#}"),
946 }
947 .into())
948 }
949 }
950 }
951
952 async fn inherit(
957 repo: &Path,
958 branch: &str,
959 base_commit: &str,
960 takeover: &crate::handover::Takeover,
961 ) -> Option<crate::recovery::RecoveryRecord> {
962 use crate::recovery::{Relation, assess};
963 let newest = takeover.earlier.last()?;
964 let mut chosen = None;
965 for id in takeover.earlier.iter().rev() {
966 if let Ok(old) = RunState::load_under(id, &takeover.home)
967 && !old.reviews.is_empty()
968 && (old.winner().is_some_and(|w| w.branch == branch)
969 || old.pr_head.as_deref() == Some(branch))
970 {
971 chosen = Some(old);
972 break;
973 }
974 }
975 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976 .await
977 .unwrap_or_default();
978 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979 Some(last) if last.head == tip => Relation::Same,
980 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981 _ => Relation::Diverged,
982 };
983 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984 Some(assess(from, chosen.as_ref(), base_commit, relation))
985 }
986
987 async fn open_review(
990 repo: &Path,
991 branch: &str,
992 mut state: RunState,
993 roles: ResolvedRoles,
994 max_parallel: usize,
995 base_commit: String,
996 takeover: &crate::handover::Takeover,
997 ) -> Result<Self> {
998 let wait = repository_wait(&state, None);
999 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000 if state.pr_recovery.is_none() {
1001 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002 }
1003 let start = review_base(
1006 repo,
1007 &state.config.merge.remote,
1008 &state.base_branch,
1009 &base_commit,
1010 branch,
1011 )
1012 .await;
1013 let log = git::log_oneline(repo, &start, branch)
1014 .await
1015 .unwrap_or_default();
1016 let log_text = if log.trim().is_empty() {
1017 "(no commit messages)"
1018 } else {
1019 log.trim()
1020 };
1021 let mut instruction = format!(
1022 "Review the work already on branch `{branch}`. There is no task \
1023 statement: what the change claims to do is whatever its commits \
1024 say.\n\n{log_text}"
1025 );
1026 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030 if let Some(task) = takeover
1031 .task_instruction
1032 .as_deref()
1033 .filter(|t| !t.trim().is_empty())
1034 && rec.reviewed_head.is_some()
1035 {
1036 instruction = format!(
1037 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038 task.trim()
1039 );
1040 rec.criteria_inherited = true;
1041 }
1042 state.event("recovery", rec.summary());
1043 state.recovery = Some(rec);
1044 }
1045 if let Some(selected) = &state.pr_recovery {
1046 instruction = selected.instruction.clone();
1047 }
1048 state.task_instruction = takeover.task_instruction.clone();
1049 state.instruction = instruction;
1050 state.reviewed_commits = Some(
1051 git::subjects(repo, &start, branch)
1052 .await
1053 .unwrap_or_default(),
1054 );
1055
1056 for id in &takeover.earlier {
1059 if let Ok(old) = RunState::load_under(id, &takeover.home)
1060 && old.repo == state.repo
1061 && (old.pr_head.as_deref() == Some(branch)
1062 || old.winner().is_some_and(|w| w.branch == branch))
1063 {
1064 for hold in old.branch_owner_holds {
1065 if !state.branch_owner_holds.contains(&hold) {
1066 state.event(
1067 "approval-policy",
1068 format!(
1069 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070 hold.control.key(),
1071 hold.file,
1072 hold.head
1073 ),
1074 );
1075 state.branch_owner_holds.push(hold);
1076 }
1077 }
1078 }
1079 }
1080 let reviewed_head = git::rev_parse(repo, branch).await?;
1081 state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083 let worktree = state.worktree_root().join("under-review");
1086 if let Some(parent) = worktree.parent() {
1087 tokio::fs::create_dir_all(parent).await.ok();
1088 }
1089 let path = worktree.to_string_lossy().to_string();
1090 git::git(repo, &["worktree", "add", &path, branch])
1091 .await
1092 .with_context(|| {
1093 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094 })?;
1095
1096 let setup = match ensure_setup_config(&mut state, repo).await {
1097 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098 Err(e) => Err(e),
1099 };
1100 if let Err(e) = setup {
1101 git::worktree_remove_clean(repo, &worktree).await.ok();
1102 return Err(e);
1103 }
1104 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105 .await
1106 .unwrap_or(0);
1107 if commits == 0 {
1108 git::worktree_remove_clean(repo, &worktree).await.ok();
1109 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110 }
1111 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112 .await
1113 .map(|f| f.len())
1114 .unwrap_or(0);
1115 if files == 0
1116 && let (Ok(head_tree), Ok(base_tree)) = (
1117 git::tree_of(&worktree, "HEAD").await,
1118 git::tree_of(&worktree, &base_commit).await,
1119 )
1120 && head_tree == base_tree
1121 {
1122 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123 git::worktree_remove_clean(repo, &worktree).await.ok();
1124 bail!(
1125 "`{branch}` at {} has a tree identical to base {}; this usually means \
1126 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127 against `{}/{branch}`) rather than an empty change",
1128 short(&head),
1129 short(&base_commit),
1130 state.config.merge.remote
1131 );
1132 }
1133 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134 .await
1135 .unwrap_or_default();
1136
1137 state.candidates.push(Candidate {
1138 index: 0,
1139 label: 'A',
1140 agent: EXISTING_BRANCH.to_owned(),
1143 branch: branch.to_owned(),
1144 worktree,
1145 summary: String::new(),
1146 stat,
1147 files,
1148 commits,
1149 empty: false,
1150 failed: None,
1151 verified_noop: None,
1152 duration_ms: 0,
1153 folded: false,
1154 });
1155 state.tally = Some(Tally {
1156 first_choice: BTreeMap::from([('A', 0)]),
1157 borda: BTreeMap::new(),
1158 winner: 'A',
1159 rankings: 0,
1160 unanimous_initial: false,
1161 deliberated: false,
1162 changed_votes: 0,
1163 unanimous_final: false,
1164 tie_break: None,
1165 judges: 0,
1169 present: 0,
1170 quorum: 0,
1171 met_quorum: true,
1172 uncontested: Some("review-only run: nothing competed".to_owned()),
1173 });
1174 state.status = RunStatus::Reviewing;
1175 state.prep_complete = Some(true);
1176 state.event(
1177 "start",
1178 format!(
1179 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180 state.id
1181 ),
1182 );
1183 state.save()?;
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub fn resume(id: &str) -> Result<Self> {
1195 let state = RunState::load(id)?;
1196 crate::pr_head::ensure_owner(&state)?;
1197 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198 bail!(
1199 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200 );
1201 }
1202 if let Some(to) = &state.released_to {
1203 bail!(
1204 "run {} cannot be resumed: its worktree was released to run {}",
1205 state.short(),
1206 crate::run::short_of(to)
1207 );
1208 }
1209 let roles = state.config.resolve_roles()?;
1210 if let Some(id) = &state.config.graph.selected_implementer
1211 && state.candidates.iter().any(|c| c.agent != *id)
1212 {
1213 bail!(
1214 "replacement implementer pin does not match the saved candidates; task must stay held"
1215 );
1216 }
1217 let max_parallel = state.config.graph.max_parallel.max(1);
1218 Ok(Self {
1219 state,
1220 roles,
1221 sem: Arc::new(Semaphore::new(max_parallel)),
1222 pause: Pause::new(),
1223 interrupt: Pause::new(),
1224 })
1225 }
1226
1227 pub async fn execute(&mut self) -> Result<()> {
1234 if let Some(selection) = &self.state.pr_recovery
1235 && (selection.source_proof.is_some() || selection.source_run.is_some())
1236 {
1237 Box::pin(selection.verify_source(&self.state.repo)).await?;
1238 }
1239 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242 || self.state.land_approval.is_some()
1243 || self.state.merge_decision.is_some()
1244 || !self.state.merge_repairs.is_empty()
1245 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246 {
1247 let claim = FixClaim::acquire(&self.state.dir())?;
1248 let disk = RunState::load(&self.state.id)?;
1249 let checkpoint = |s: &RunState| {
1250 (
1251 s.merge_repairs.len(),
1252 s.merge_repairs
1253 .last()
1254 .map(|r| (r.question.clone(), r.phase)),
1255 s.land_approval
1256 .as_ref()
1257 .map(|a| (a.question.clone(), a.head.clone())),
1258 )
1259 };
1260 if checkpoint(&disk) != checkpoint(&self.state) {
1261 bail!(
1262 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263 );
1264 }
1265 Some(claim)
1266 } else {
1267 None
1268 };
1269 let mut execution = self.claim_driver().await?;
1270 let runner = &mut *execution.runner;
1271 let wait = runner.repository_wait();
1272 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275 let result = match result {
1276 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277 record_repository_block(&mut runner.state, &e);
1278 runner.park_here()?;
1279 Ok(())
1280 }
1281 other => other,
1282 };
1283 execution.driver.finish(&mut runner.state).await?;
1284 let ended = if result.is_err() {
1285 Some({
1286 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1287 if let Err(error) = &result {
1288 notice.message = format!(
1289 "Run {} stopped with an error: {}",
1290 runner.state.id,
1291 error.to_string().lines().next().unwrap_or("unknown error")
1292 );
1293 }
1294 notice
1295 })
1296 } else {
1297 crate::notices::run_ended(&runner.state)
1298 };
1299 if let Some(notice) = ended {
1300 crate::notices::raise(notice);
1301 }
1302 result
1303 }
1304
1305 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1306 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1307 }
1308
1309 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1310 let wait = self.repository_wait();
1311 coordinate_operation(&mut self.state, cwd, &wait).await
1312 }
1313
1314 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1316 let owner = match crate::pr_head::owner_lock_wait(
1317 &crate::run::home(),
1318 &self.state.id,
1319 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1320 )
1321 .await
1322 {
1323 Ok(owner) => owner,
1324 Err(error) => {
1325 let detail = format!("worktree maintenance or ownership update: {error:#}");
1326 self.state.resource_block = Some(crate::run::ResourceBlock {
1330 cause: crate::run::ResourceBlockCause::Other,
1331 node: "start".into(),
1332 kind: "worktree maintenance".into(),
1333 detail: detail.clone(),
1334 at: Timestamp::now(),
1335 });
1336 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1337 }
1338 };
1339 let disk = RunState::load(&self.state.id)?;
1340 crate::pr_head::ensure_owner(&disk)?;
1341 let claimed =
1342 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1343 anyhow::ensure!(
1348 disk.liveness(claimed) != crate::run::Liveness::Live,
1349 "run already has a live driver; wait for its execution to end"
1350 );
1351 if let Some(to) = &disk.released_to {
1352 bail!(
1353 "run {} cannot continue: its worktree was released to run {}",
1354 self.state.short(),
1355 crate::run::short_of(to)
1356 );
1357 }
1358 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1359 let mut execution = DriverExecution {
1360 runner: self,
1361 driver,
1362 };
1363 let _owner = owner;
1366 let state = &mut execution.runner.state;
1367 state.parked = false;
1372 state.resource_wait = None;
1373 state.resource_block = None;
1374 state.clear_active();
1381 let pid = std::process::id();
1396 state.driver_pid = Some(pid);
1397 state.driver_started_at = crate::proc::process_started_at(pid);
1398 state.driver_exited = false;
1399 state.driver_claimed = Some(true);
1400 state.begin_pass();
1401 execution.driver.bind(state);
1402 state.save()?;
1403 execution.driver.record(state)?;
1404 Ok(execution)
1405 }
1406
1407 async fn execute_graph(&mut self) -> Result<()> {
1408 self.state.configure_recovery_resume()?;
1409 self.persist_candidates().await?;
1412 if self
1413 .state
1414 .merge_decision
1415 .as_ref()
1416 .is_some_and(|decision| !decision.closed)
1417 && self.state.status != RunStatus::Landing
1418 {
1419 if crate::merge_decision::resumable_hold(&self.state) {
1420 self.state.merge = Some(MergeOutcome {
1421 mode: MergeMode::Pr,
1422 ok: false,
1423 detail: self
1424 .state
1425 .merge_decision
1426 .as_ref()
1427 .unwrap()
1428 .target
1429 .pr
1430 .clone(),
1431 empty: false,
1432 });
1433 self.run_land().await?;
1434 self.settle_questions();
1435 return Ok(());
1436 }
1437 bail!(
1438 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1439 );
1440 }
1441 if self.state.config.graph.selected_implementer.is_some()
1442 && self.state.status == RunStatus::Blocked
1443 && self.state.tally.is_none()
1444 && self.state.candidate_preservation.is_empty()
1445 && self.state.interrupted_implementations.is_empty()
1446 {
1447 bail!(
1448 "selected implementation is held; explicitly release the owning task to retry it"
1449 );
1450 }
1451
1452 if self.state.merge_repairs.last().is_some_and(|r| {
1453 matches!(
1454 r.phase,
1455 crate::run::MergeRepairPhase::Requested
1456 | crate::run::MergeRepairPhase::Fixing
1457 | crate::run::MergeRepairPhase::Reviewing
1458 )
1459 }) {
1460 if self.repair_merge_review().await? {
1461 self.run_land().await?;
1462 }
1463 self.settle_questions();
1464 return Ok(());
1465 }
1466 if self.state.pr_publication_pending {
1469 self.merge().await?;
1470 if self.resource_halt()? {
1471 return Ok(());
1472 }
1473 self.state.save()?;
1474 self.settle_questions();
1475 return Ok(());
1476 }
1477 if self.state.status == RunStatus::Stalled {
1490 if self.recover_stall().await? {
1491 self.finish_after_tally().await?;
1492 } else {
1493 self.state.save()?;
1495 }
1496 return Ok(());
1497 }
1498 if self.state.status == RunStatus::Landing
1508 || crate::handover::landing_checkpoint(&self.state)
1509 {
1510 self.state.land_sync_pending = true;
1511 self.state.save()?;
1512 self.run_land().await?;
1513 self.settle_questions();
1518 return Ok(());
1519 }
1520 if self.state.github_text.is_some()
1525 && self.state.merge.is_none()
1526 && self.state.status == RunStatus::Gating
1527 {
1528 self.merge().await?;
1529 if self.resource_halt()? {
1530 return Ok(());
1531 }
1532 self.state.save()?;
1533 return Ok(());
1534 }
1535 if let Err(e) = self.prep().await {
1536 if e.is::<crate::repo_lock::ResourceBlocked>() {
1537 return Err(e);
1538 }
1539 self.state.status = RunStatus::Blocked;
1540 self.state
1541 .prep_failure
1542 .get_or_insert_with(|| format!("{e:#}"));
1543 self.state
1544 .event("prep", format!("preparation stopped: {e:#}"));
1545 self.state.save()?;
1546 return Err(e);
1547 }
1548 if self.park_here()? {
1549 return Ok(());
1550 }
1551 self.advise().await?;
1552 if self.park_here()? {
1553 return Ok(());
1554 }
1555 self.implement().await?;
1556 if self.park_here()? {
1557 return Ok(());
1558 }
1559 if self.state.status == RunStatus::VerifiedNoop {
1563 return Ok(());
1564 }
1565 self.judge().await?;
1566 if self.park_here()? {
1567 return Ok(());
1568 }
1569 self.deliberate().await?;
1570 if self.park_here()? {
1571 return Ok(());
1572 }
1573 self.vote().await?;
1574 if self.park_here()? {
1575 return Ok(());
1576 }
1577 self.tally()?;
1578 if self.state.status == RunStatus::Stalled {
1583 self.state.save()?;
1587 return Ok(());
1588 }
1589 self.finish_after_tally().await?;
1590 Ok(())
1591 }
1592
1593 fn stop_for_resource(&mut self) -> Result<()> {
1598 if self.pause.parked() || self.interrupt.parked() {
1601 self.park_here()?;
1602 return Ok(());
1603 }
1604 if self.state.resource_block.is_some() {
1605 self.state.status = RunStatus::Blocked;
1606 }
1607 self.state.save()
1608 }
1609
1610 fn resource_halt(&mut self) -> Result<bool> {
1612 if self.state.resource_block.is_none() {
1613 return Ok(false);
1614 }
1615 self.stop_for_resource()?;
1616 Ok(true)
1617 }
1618
1619 fn park_here(&mut self) -> Result<bool> {
1626 let parking = self.pause.parked() || self.interrupt.parked();
1627 if !parking && self.state.resource_block.is_some() {
1628 self.stop_for_resource()?;
1629 return Ok(true);
1630 }
1631 if !self.pause.parked() && !self.interrupt.parked() {
1636 return Ok(false);
1637 }
1638 if self.state.parked {
1639 return Ok(true);
1640 }
1641 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1642 if let Some(block) = &mut self.state.resource_block
1643 && matches!(
1644 block.cause,
1645 crate::run::ResourceBlockCause::Cancelled { .. }
1646 )
1647 {
1648 block.cause = crate::run::ResourceBlockCause::Cancelled {
1649 park: cause.clone(),
1650 };
1651 }
1652 let why = match cause.as_ref().map(ParkCause::label) {
1653 Some(reason) => format!(
1654 "parked after `{}` ({reason}) — resume to carry on from here",
1655 self.state.status.as_str()
1656 ),
1657 None => format!(
1658 "parked after `{}` — resume to carry on from here",
1659 self.state.status.as_str()
1660 ),
1661 };
1662 self.state.event("park", why);
1663 self.state
1664 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1665 self.state.parked = true;
1666 self.state.save()?;
1667 Ok(true)
1668 }
1669
1670 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1672 if self.state.prep_failure.is_none() {
1673 return Ok(());
1674 }
1675 let failed_at = self
1676 .state
1677 .events
1678 .iter()
1679 .rev()
1680 .find(|event| event.node == "prep")
1681 .map_or(self.state.updated_at, |event| event.at);
1682 anyhow::ensure!(
1683 approval_at > failed_at,
1684 "preparation retry requires a new explicit release after the failure"
1685 );
1686 self.state.prep_failure = None;
1687 self.state.prep_complete = Some(false);
1688 self.state
1689 .event("prep-retry", "operator approved retry of preparation");
1690 Ok(())
1691 }
1692
1693 pub fn on_pause(&mut self, pause: Pause) {
1695 self.pause = pause;
1696 }
1697
1698 pub fn watch_interrupt(&mut self, pause: Pause) {
1704 self.interrupt = pause;
1705 }
1706
1707 fn settle_questions(&mut self) {
1727 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1728 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1729 }
1730 }
1731
1732 async fn finish_after_tally(&mut self) -> Result<()> {
1735 self.graft().await?;
1738 if self.resource_halt()? {
1739 return Ok(());
1740 }
1741 self.fold_losers().await?;
1742 self.sync_to_base().await?;
1747 if self.resource_halt()? {
1748 return Ok(());
1749 }
1750 if self.state.status == RunStatus::AlreadyInBase {
1751 return Ok(());
1752 }
1753 self.review_loop().await?;
1754 if self.resource_halt()? {
1755 return Ok(());
1756 }
1757 self.sync_to_base().await?;
1758 if self.resource_halt()? {
1759 return Ok(());
1760 }
1761 if self.state.status == RunStatus::AlreadyInBase {
1762 return Ok(());
1763 }
1764 self.gate().await?;
1765 self.merge().await?;
1766 if self.resource_halt()? {
1767 return Ok(());
1768 }
1769 self.state.save()?;
1770 Ok(())
1771 }
1772
1773 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1780 let result = match ensure_setup_config(&mut self.state, repo).await {
1781 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1782 Err(e) => Err(e),
1783 };
1784 if let Err(e) = result {
1785 git::worktree_remove_clean(repo, wt).await.ok();
1786 if in_prep {
1787 self.rollback_prep(repo).await;
1788 }
1789 self.state.status = RunStatus::Blocked;
1790 self.state.event(
1791 "setup",
1792 format!("worktree setup failed in {}: {e:#}", wt.display()),
1793 );
1794 self.state.save()?;
1795 return Err(e);
1796 }
1797 Ok(())
1798 }
1799
1800 async fn rollback_prep(&mut self, repo: &Path) {
1802 let root = self.state.worktree_root();
1803 for c in std::mem::take(&mut self.state.candidates) {
1804 if matches!(
1805 git::worktree_remove_clean(repo, &c.worktree).await,
1806 Ok(true)
1807 ) {
1808 git::branch_delete(repo, &c.branch).await.ok();
1809 } else {
1810 self.state.candidates.push(c);
1811 }
1812 }
1813 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1814 for j in 1..=seats {
1815 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1816 .await
1817 .ok();
1818 }
1819 for k in 1..=self.state.config.graph.advisors {
1820 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1821 .await
1822 .ok();
1823 }
1824 }
1825
1826 async fn prep(&mut self) -> Result<()> {
1827 if let Some(reason) = &self.state.prep_failure {
1828 bail!(
1829 "candidate preparation previously failed: {reason}; correct the source before retrying"
1830 );
1831 }
1832 if self
1833 .state
1834 .prep_complete
1835 .unwrap_or(!self.state.candidates.is_empty())
1836 {
1837 return Ok(());
1838 }
1839 let repo = self.state.repo.clone();
1840 let _coordination = self.coordinate(&repo).await?;
1843 let base = self.state.base_commit.clone();
1844 let plan = self.state.sources.plan(&repo, &base).await?;
1845 self.rollback_prep(&repo).await;
1846 if !self.state.candidates.is_empty() {
1847 bail!(
1848 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1849 );
1850 }
1851 self.state.prep_complete = Some(false);
1852 self.state.status = RunStatus::Prep;
1853 self.state.save()?;
1854 let repo = self.state.repo.clone();
1855 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1856 let root = self.state.worktree_root();
1857 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1858
1859 let hooks_dir = self.state.dir().join("hooks");
1862 if self.state.config.blind.commit_msg_hook {
1863 std::fs::create_dir_all(&hooks_dir)
1864 .with_context(|| format!("create {}", hooks_dir.display()))?;
1865 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1866 let path = hooks_dir.join("commit-msg");
1867 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1868 make_executable(&path)?;
1869 git::acquire_worktree_config(&repo).await?;
1877 self.state.enabled_worktree_config = true;
1878 }
1879
1880 ensure_setup_config(&mut self.state, &repo).await?;
1883
1884 for (index, (spec, label)) in self
1885 .roles
1886 .implementers
1887 .clone()
1888 .into_iter()
1889 .zip(labels)
1890 .enumerate()
1891 {
1892 let branch = self.state.branch_for(label);
1893 let worktree = root.join(format!("cand-{label}"));
1894 self.state.event(
1898 "prep-intent",
1899 serde_json::json!({
1900 "branch": branch,
1901 "worktree": worktree,
1902 "start": start,
1903 "picks": plan.picks,
1904 })
1905 .to_string(),
1906 );
1907 self.state.save()?;
1908 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1909 if self.state.config.blind.commit_msg_hook {
1910 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1911 }
1912 git::local_exclude(&worktree, "/.magi/").await?;
1913 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1914 git::worktree_remove_clean(&repo, &worktree).await.ok();
1917 git::branch_delete(&repo, &branch).await.ok();
1918 self.rollback_prep(&repo).await;
1919 self.state.status = RunStatus::Blocked;
1920 self.state
1921 .event("prep", format!("worktree setup failed: {e:#}"));
1922 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1923 self.state.save()?;
1924 return Err(e);
1925 }
1926 self.state.candidates.push(Candidate {
1927 index,
1928 label,
1929 agent: spec.id.clone(),
1930 branch,
1931 worktree,
1932 summary: String::new(),
1933 stat: String::new(),
1934 files: 0,
1935 commits: 0,
1936 empty: false,
1937 failed: None,
1938 verified_noop: None,
1939 duration_ms: 0,
1940 folded: false,
1941 });
1942 self.state.save()?;
1943 for pick in &plan.picks {
1944 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1945 {
1946 self.state.status = RunStatus::Blocked;
1947 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1948 self.state
1949 .event("prep", format!("cannot apply referenced commit: {e}"));
1950 self.state.save()?;
1951 return Err(e);
1952 }
1953 }
1954 }
1955
1956 for j in 1..=self.roles.judges.len() {
1957 let wt = root.join(format!("judge-{j}"));
1958 if !wt.exists() {
1959 git::worktree_add_detached(&repo, &wt, &base).await?;
1960 self.setup_seat_worktree(&repo, &wt, true).await?;
1961 }
1962 }
1963
1964 if self.state.config.graph.advise {
1972 for k in 1..=self.state.config.graph.advisors {
1973 let wt = root.join(format!("advisor-{k}"));
1974 if !wt.exists() {
1975 git::worktree_add_detached(&repo, &wt, &base).await?;
1976 self.setup_seat_worktree(&repo, &wt, true).await?;
1977 }
1978 }
1979 }
1980
1981 let authors: Vec<&str> = self
1986 .roles
1987 .implementers
1988 .iter()
1989 .map(|a| a.id.as_str())
1990 .collect();
1991 let overlap: Vec<String> = self
1992 .roles
1993 .judges
1994 .iter()
1995 .enumerate()
1996 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1997 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1998 .collect();
1999 if !overlap.is_empty() {
2000 let note = format!(
2001 "{} also authored a candidate; blind, but the panel is less \
2002 independent than {} distinct agents would be",
2003 overlap.join(", "),
2004 self.roles.judges.len()
2005 );
2006 self.state.event("prep", note);
2007 }
2008
2009 self.state.event(
2010 "prep",
2011 format!(
2012 "{} candidates, {} judges, base {} ({})",
2013 self.state.candidates.len(),
2014 self.roles.judges.len(),
2015 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2016 self.state.base_branch
2017 ),
2018 );
2019 self.state.prep_complete = Some(true);
2020 self.state.status = RunStatus::Implementing;
2021 self.state.save()?;
2022 Ok(())
2023 }
2024
2025 async fn advise(&mut self) -> Result<()> {
2058 let implement_untouched = self
2059 .state
2060 .candidates
2061 .iter()
2062 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2063 if !self.state.config.graph.advise || self.state.advise_attempted {
2064 return Ok(());
2065 }
2066 if !implement_untouched {
2067 self.state.event(
2068 "advise",
2069 "skipping the design-deliberation stage: at least one \
2070 candidate already shows implementation progress, so this \
2071 run is past the point the stage exists to run before"
2072 .to_owned(),
2073 );
2074 self.state.advise_attempted = true;
2075 self.state.save()?;
2076 return Ok(());
2077 }
2078 let run_id = self.state.id.clone();
2079 let prompts = self.state.config.prompts.clone();
2080 let instruction = self.state.instruction.clone();
2081 let language = self.state.config.graph.language.clone();
2082 let root = self.state.worktree_root();
2083 let n = self.state.config.graph.advisors;
2084 let where_recorded = self.state.dir().join("run.json");
2085
2086 let seats = match self.state.config.advisors() {
2087 Ok(seats) if !seats.is_empty() => seats,
2088 Ok(_) => {
2089 self.state.event(
2090 "advise",
2091 format!(
2092 "[graph] advisors is 0; skipping the design-deliberation \
2093 stage and continuing without a synthesis brief (see {})",
2094 where_recorded.display()
2095 ),
2096 );
2097 self.state.advise_attempted = true;
2098 self.state.save()?;
2099 return Ok(());
2100 }
2101 Err(e) => {
2102 self.state.event(
2103 "advise",
2104 format!(
2105 "could not resolve advisor seats ({e:#}); continuing \
2106 without a design-deliberation brief (see {})",
2107 where_recorded.display()
2108 ),
2109 );
2110 self.state.advise_attempted = true;
2111 self.state.save()?;
2112 return Ok(());
2113 }
2114 };
2115
2116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2117 let artifacts = agent::artifacts_dir(&self.state.dir());
2118 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2119
2120 let mut jobs = Vec::new();
2121 for (i, spec) in seats.iter().cloned().enumerate() {
2122 let seat_key = format!("advisor-{}", i + 1);
2123 let seat = self.seat(&seat_key, &spec.id);
2124 jobs.push(SeatJob {
2125 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2126 spec,
2127 seat,
2128 cwd: worktrees[i % worktrees.len()].clone(),
2129 timeout,
2130 allow_write: false,
2131 sessions: false,
2132 artifacts: artifacts.clone(),
2133 stem: seat_key,
2134 handover: None,
2135 });
2136 }
2137
2138 self.state.event(
2139 "advise",
2140 format!(
2141 "{} advisor seat(s) sketching a design in parallel",
2142 jobs.len()
2143 ),
2144 );
2145 let mut quota_losses = Vec::new();
2146 let cache = self.state.config.cache_dir();
2147 let ctx = WaveCtx {
2148 carry_seats: false,
2149 pauses: Some((&self.pause, &self.interrupt)),
2150 run: &run_id,
2151 node: "advise",
2152 prompts: &prompts,
2153 cache: cache.as_deref(),
2154 round: None,
2155 };
2156 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2157 let results = ask_json_wave::<Proposal>(
2158 jobs,
2159 Arc::clone(&self.sem),
2160 self.state.config.graph.retries,
2161 &advisor_roster,
2162 &ctx,
2163 &mut quota_losses,
2164 &mut self.state,
2165 &|p: &Proposal| p.validate(),
2166 )
2167 .await;
2168 if self.state.resource_block.is_some() {
2169 return self.stop_for_resource();
2170 }
2171 self.state.quota.extend(quota_losses);
2172
2173 let mut records = Vec::with_capacity(results.len());
2174 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2175 let agent_id = seat.agent.clone();
2176 self.state.seats.insert(seat.key.clone(), seat);
2177 match res {
2178 Ok((proposal, out)) => {
2179 self.state
2180 .event("advise", format!("advisor-{} proposed a design", i + 1));
2181 records.push(advise::AdvisorRecord::proposed(
2182 i + 1,
2183 agent_id,
2184 proposal,
2185 out.duration_ms,
2186 ));
2187 }
2188 Err(e) => {
2189 self.state.event(
2190 "advise",
2191 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2192 );
2193 records.push(advise::AdvisorRecord::failed(
2194 i + 1,
2195 agent_id,
2196 e.to_string(),
2197 ));
2198 }
2199 }
2200 }
2201
2202 let mut advice = advise::Advice {
2203 records,
2204 synthesis: None,
2205 };
2206 if advice.proposals().is_empty() {
2207 self.state.event(
2208 "advise",
2209 "no advisor produced a usable proposal; continuing without a \
2210 synthesis brief"
2211 .to_owned(),
2212 );
2213 } else {
2214 match self
2215 .synthesize_brief(
2216 &advice,
2217 &instruction,
2218 &language,
2219 &worktrees[0],
2220 &artifacts,
2221 &run_id,
2222 &prompts,
2223 cache.as_deref(),
2224 )
2225 .await
2226 {
2227 Ok(Some(text)) => {
2228 self.state.event(
2229 "advise",
2230 "synthesized a design brief for the implementer".to_owned(),
2231 );
2232 advice.synthesis = Some(text);
2233 }
2234 Ok(None) => {
2235 self.state.event(
2236 "advise",
2237 "the synthesis seat produced nothing usable; continuing \
2238 without a design brief"
2239 .to_owned(),
2240 );
2241 }
2242 Err(e) => {
2243 if e.is::<crate::repo_lock::ResourceBlocked>() {
2244 return Err(e);
2245 }
2246 self.state.event(
2247 "advise",
2248 format!("could not synthesize a design brief: {e:#}"),
2249 );
2250 }
2251 }
2252 }
2253 advise::apply_reflection(&mut advice);
2254
2255 self.state.advice = Some(advice);
2256 self.state.advise_attempted = true;
2257 self.state.save()?;
2258 Ok(())
2259 }
2260
2261 #[allow(clippy::too_many_arguments)]
2273 async fn synthesize_brief(
2274 &mut self,
2275 advice: &advise::Advice,
2276 instruction: &str,
2277 language: &str,
2278 cwd: &Path,
2279 artifacts: &Path,
2280 run_id: &str,
2281 prompts: &Prompts,
2282 cache: Option<&Path>,
2283 ) -> Result<Option<String>> {
2284 let chain = agent::pick_chain(
2285 &self.state.config.agents,
2286 self.state.config.roles.synthesizer.as_ref(),
2287 &agent::installed,
2288 "synthesizer",
2289 )?;
2290 let proposals = advice.proposals();
2291 let mut prompt = prompt::with_overlay(
2292 prompt::synthesize_brief(instruction, &proposals, language),
2293 prompts.overlay("advise"),
2294 );
2295 if cache.is_some() {
2296 prompt.push('\n');
2301 prompt.push_str(&prompt::build_cache_note("advise", false));
2302 }
2303 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2304 let mut last = None;
2309 for (n, spec) in chain.iter().enumerate() {
2310 if n > 0 {
2311 self.state
2312 .event("advise", format!("synthesis falling back to {}", spec.id));
2313 }
2314 let mut seat = self.seat("advise-synthesis", &spec.id);
2315 let wait = self.repository_wait();
2316 let outcome = wait
2317 .scope(agent::invoke(
2318 spec,
2319 &mut seat,
2320 &Invocation {
2321 coordination: self.state.config.coordination.as_ref(),
2322 editing_network: self.state.config.graph.editing_network,
2323 fetched_base: None,
2324 cwd,
2325 prompt: &prompt,
2326 timeout,
2327 allow_write: false,
2328 unsandboxed: false,
2329 sessions: false,
2330 artifacts,
2331 stem: &if n == 0 {
2332 "advise-synthesis".to_owned()
2333 } else {
2334 format!("advise-synthesis-{}", spec.id)
2335 },
2336 run: run_id,
2337 node: "advise",
2338 cache_dir: None,
2339 attachments: &[],
2340 writable: &[],
2341 },
2342 ))
2343 .await;
2344 if outcome.is_ok() {
2345 self.state.seats.insert(seat.key.clone(), seat);
2346 }
2347 let advance = agent::chain_advances(&outcome);
2348 last = Some(outcome);
2349 if !advance {
2350 break;
2351 }
2352 }
2353 let out = last.expect("a chain holds at least one agent")?;
2356 if !out.usable() {
2357 return Ok(None);
2358 }
2359 let text =
2360 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2361 Ok((!text.trim().is_empty()).then_some(text))
2362 }
2363
2364 async fn implement(&mut self) -> Result<()> {
2367 self.persist_candidates().await?;
2368 let run_id = self.state.id.clone();
2373 let prompts = self.state.config.prompts.clone();
2374 let todo: Vec<usize> = self
2375 .state
2376 .candidates
2377 .iter()
2378 .enumerate()
2379 .filter(|(_, c)| {
2380 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2381 && c.failed.is_none()
2382 && !c.empty
2383 })
2384 .map(|(i, _)| i)
2385 .collect();
2386 if todo.is_empty() {
2387 return self.after_implement();
2388 }
2389 self.state.status = RunStatus::Implementing;
2390
2391 let language = self.state.config.graph.language.clone();
2392 let github_language = self.state.config.graph.github_language.clone();
2393 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2394 let sessions = self.state.config.graph.sessions;
2395 let artifacts = agent::artifacts_dir(&self.state.dir());
2396 let brief = self
2400 .state
2401 .advice
2402 .as_ref()
2403 .and_then(|a| a.synthesis.as_deref())
2404 .map(str::to_owned);
2405 let attachments = self.state.attachments.clone();
2406
2407 let mut jobs = Vec::new();
2408 for &i in &todo {
2409 let (index, label, worktree) = {
2410 let c = &self.state.candidates[i];
2411 (c.index, c.label, c.worktree.clone())
2412 };
2413 let spec = self.roles.implementers[index].clone();
2414 let seat_key = format!("impl-{label}");
2415 let seat = self.seat(&seat_key, &spec.id);
2416 let instruction = seeded_instruction(&self.state);
2417 jobs.push(SeatJob {
2418 spec,
2419 seat,
2420 prompt: prompt::implement_in(
2421 &instruction,
2422 &worktree.to_string_lossy(),
2423 &language,
2424 &github_language,
2425 brief.as_deref(),
2426 &attachments,
2427 ),
2428 cwd: worktree,
2429 timeout,
2430 allow_write: true,
2431 sessions,
2432 artifacts: artifacts.clone(),
2433 stem: format!("impl-{label}"),
2434 handover: None,
2435 });
2436 }
2437
2438 self.state.event(
2439 "implement",
2440 format!("{} candidates in parallel", jobs.len()),
2441 );
2442 let mut sent = jobs.clone();
2448 let cache = self.state.config.cache_dir();
2449 let ctx = WaveCtx {
2450 carry_seats: false,
2451 pauses: Some((&self.pause, &self.interrupt)),
2452 run: &run_id,
2453 node: "implement",
2454 prompts: &prompts,
2455 cache: cache.as_deref(),
2456 round: None,
2457 };
2458 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2459 if !wave_blocked(&results) {
2460 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2461 .await;
2462 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2463 .await?;
2464 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2465 .await;
2466 }
2467 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2468
2469 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2470 && results
2471 .iter()
2472 .any(|(_, _, out)| FailClass::of(out).is_some());
2473 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2474 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2475 if self
2479 .state
2480 .candidate_preservation
2481 .contains_key(&self.state.candidates[i].label)
2482 {
2483 self.state.seats.insert(seat.key.clone(), seat);
2484 }
2485 continue;
2486 }
2487 let seat_key = seat.key.clone();
2488 let agent = seat.agent.clone();
2497 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2498 self.state.seats.insert(seat.key.clone(), seat);
2499 let label = self.state.candidates[i].label;
2500 self.state.interrupted_implementations.remove(&label);
2501
2502 let (summary, duration, failed, verified_claim) = match out {
2503 AgentOutcome::Ok(o) => {
2504 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2505 let failed = (!o.usable()).then(|| {
2506 if o.timed_out {
2507 "agent timed out".to_owned()
2508 } else {
2509 format!(
2510 "agent exited with {}",
2511 crate::notices::exit_status(o.exit_code)
2512 )
2513 }
2514 });
2515 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2516 (text, o.duration_ms, failed, verified_claim)
2517 }
2518 AgentOutcome::Dropped(o) => {
2524 let why = o
2525 .dropped
2526 .as_ref()
2527 .map(|d| d.why.as_str())
2528 .unwrap_or("the CLI ended the stream without delivering its answer");
2529 (
2530 String::new(),
2531 o.duration_ms,
2532 Some(format!("the CLI dropped the stream ({why})")),
2533 None,
2534 )
2535 }
2536 AgentOutcome::Quota(o) => {
2537 self.state.quota.push(QuotaLoss {
2538 seat: seat_key,
2539 node: "implement".to_owned(),
2540 at: Timestamp::now(),
2541 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2542 });
2543 (
2544 String::new(),
2545 o.duration_ms,
2546 Some("rate limited (quota); produced no change".to_owned()),
2547 None,
2548 )
2549 }
2550 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2551 (String::new(), 0, Some(e), None)
2552 }
2553 };
2554
2555 let c = &mut self.state.candidates[i];
2558 if !exhausted_the_fallback_chain {
2559 c.agent = agent;
2560 }
2561 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2562 c.duration_ms = duration;
2563 c.failed = failed;
2564 c.verified_noop = verified_claim;
2565 self.state
2566 .candidate_preservation
2567 .insert(label, "awaiting persistence".to_owned());
2568 }
2569 self.state.save()?;
2570 self.persist_candidates().await?;
2571
2572 if resource_blocked {
2573 return self.stop_for_resource();
2574 }
2575 if selected_failed {
2576 self.state.status = RunStatus::Blocked;
2577 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2578 self.state.save()?;
2579 bail!(
2580 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2581 );
2582 }
2583 self.after_implement()
2584 }
2585
2586 async fn persist_candidates(&mut self) -> Result<()> {
2588 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2589 for label in labels {
2590 let i = self
2591 .state
2592 .candidates
2593 .iter()
2594 .position(|c| c.label == label)
2595 .context("pending candidate is missing")?;
2596 let c = self.state.candidates[i].clone();
2597 let interrupted = self.state.interrupted_implementations.contains(&label);
2598 let result = async {
2599 let wait = crate::repo_lock::WaitPolicy::new(
2600 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2601 || false,
2602 );
2603 let _coordination =
2604 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2605 if let Err(e) = git::snapshot_uncommitted(
2608 &c.worktree,
2609 &self
2610 .state
2611 .dir()
2612 .join("artifacts")
2613 .join(format!("candidate-{label}-recovery")),
2614 )
2615 .await
2616 {
2617 self.state.event(
2618 "preserve",
2619 format!("candidate {label} snapshot incomplete: {e:#}"),
2620 );
2621 }
2622 let rescued = git::rescue_commit(
2623 &c.worktree,
2624 &format!("magi: candidate {label} (uncommitted work)"),
2625 )
2626 .await?;
2627 self.state.note_withheld("implement", &rescued.withheld);
2628 let base = &self.state.base_commit;
2629 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2630 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2631 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2632 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2633 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2634 if c.empty && commits > 0 && !patch.trim().is_empty()
2635 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2636 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2637 || self.state.tally.is_some())
2638 {
2639 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2643 }
2644 let c = &mut self.state.candidates[i];
2645 c.commits = commits;
2646 c.stat = stat;
2647 c.files = files;
2648 let no_changes = commits == 0 || patch.trim().is_empty();
2649 c.empty = no_changes && !interrupted;
2650 if interrupted && no_changes {
2651 c.failed = None;
2652 c.verified_noop = None;
2653 }
2654 if !c.empty {
2655 c.failed = None;
2656 c.verified_noop = None;
2657 }
2658 self.state.event(
2659 "preserve",
2660 format!("candidate {label}: {files} files, {commits} commits"),
2661 );
2662 Ok::<_, anyhow::Error>(())
2663 }
2664 .await;
2665 match result {
2666 Ok(()) => {
2667 self.state.candidate_preservation.remove(&label);
2668 if !interrupted || self.state.candidates[i].files > 0 {
2669 self.state.interrupted_implementations.remove(&label);
2670 }
2671 }
2672 Err(e) => {
2673 let reason = format!("candidate {label} preservation failed: {e:#}");
2674 self.state.event("preserve", &reason);
2675 self.state.candidate_preservation.insert(label, reason);
2676 }
2677 }
2678 if self.state.candidate_preservation.is_empty()
2679 && self.state.status == RunStatus::Blocked
2680 && self.state.candidates.iter().all(|c| c.failed.is_none())
2681 {
2682 self.state.status = RunStatus::Implementing;
2686 self.state.event(
2687 "preserve",
2688 "candidate persistence recovered; implementation can continue",
2689 );
2690 }
2691 self.state.save()?;
2692 }
2693 if !self.state.candidate_preservation.is_empty() {
2694 self.state.status = RunStatus::Blocked;
2695 self.state.save()?;
2696 bail!(
2697 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2698 );
2699 }
2700 Ok(())
2701 }
2702
2703 async fn resume_undelivered(
2731 &mut self,
2732 results: &mut [(usize, SeatState, AgentOutcome)],
2733 sent: &[SeatJob],
2734 prompts: &Prompts,
2735 run_id: &str,
2736 ) {
2737 for (wi, seat, out) in results.iter_mut() {
2738 let Some(dropped) = (match &*out {
2739 AgentOutcome::Dropped(o) => o.dropped.clone(),
2740 _ => None,
2741 }) else {
2742 continue;
2743 };
2744 let Some(job) = sent.get(*wi) else { continue };
2745 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2747 self.state.event(
2748 "implement",
2749 format!(
2750 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2751 work is in the tree",
2752 seat.key, dropped.output_tokens, dropped.why
2753 ),
2754 );
2755 continue;
2756 }
2757 if !has_context(&job.spec, seat, job.sessions) {
2765 self.state.event(
2766 "implement",
2767 format!(
2768 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2769 is no session left to resume",
2770 seat.key, dropped.output_tokens, dropped.why
2771 ),
2772 );
2773 continue;
2774 }
2775 self.state.event(
2776 "implement",
2777 format!(
2778 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2779 conversation",
2780 seat.key, dropped.output_tokens, dropped.why
2781 ),
2782 );
2783 let mut retry = job.clone();
2784 retry.seat = seat.clone();
2785 retry.prompt = prompt::resume_after_drop(&dropped.why);
2786 retry.timeout = retry_budget(job.timeout, true);
2787 retry.stem = format!("{}-resume", job.stem);
2788 let cache = self.state.config.cache_dir();
2789 let ctx = WaveCtx {
2790 carry_seats: false,
2791 pauses: Some((&self.pause, &self.interrupt)),
2792 run: run_id,
2793 node: "implement",
2794 prompts,
2795 cache: cache.as_deref(),
2796 round: None,
2797 };
2798 let (resumed_seat, resumed) =
2799 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2800 *seat = resumed_seat;
2801 *out = resumed;
2802 }
2803 }
2804
2805 async fn resume_seat_handovers(
2870 &mut self,
2871 results: &mut [(usize, SeatState, AgentOutcome)],
2872 sent: &mut [SeatJob],
2873 prompts: &Prompts,
2874 run_id: &str,
2875 ) -> Result<()> {
2876 let instruction = seeded_instruction(&self.state);
2877 let language = self.state.config.graph.language.clone();
2878 let github_language = self.state.config.graph.github_language.clone();
2879 let brief = self
2880 .state
2881 .advice
2882 .as_ref()
2883 .and_then(|a| a.synthesis.as_deref())
2884 .map(str::to_owned);
2885 let attachments = self.state.attachments.clone();
2886 for (wi, seat, out) in results.iter_mut() {
2887 let others: BTreeSet<String> = sent
2890 .iter()
2891 .enumerate()
2892 .filter(|(j, _)| j != wi)
2893 .map(|(_, j)| j.spec.id.clone())
2894 .collect();
2895 let Some(job) = sent.get_mut(*wi) else {
2896 continue;
2897 };
2898 let start = self
2903 .roles
2904 .implementer_roster
2905 .iter()
2906 .position(|s| s.id == job.spec.id)
2907 .unwrap_or(0);
2908 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2909 let mut fallback_attempt = 0usize;
2910 let mut prev: Option<FailClass> = None;
2911 while let Some(cur) = FailClass::of(&*out) {
2912 if !should_hand_over(prev.as_ref(), &cur) {
2913 break;
2914 }
2915 let Some(next) =
2916 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2917 .cloned()
2918 else {
2919 break;
2920 };
2921 tried.insert(next.id.clone());
2922 fallback_attempt += 1;
2923
2924 let coordination = self.coordinate(&job.cwd).await?;
2925 if let Ok(r) = git::rescue_commit(
2926 &job.cwd,
2927 &format!(
2928 "magi: candidate {} (uncommitted work before {} fallback)",
2929 seat.key,
2930 if cur == FailClass::Quota {
2931 "quota"
2932 } else {
2933 "handover"
2934 }
2935 ),
2936 )
2937 .await
2938 {
2939 self.state.note_withheld("implement", &r.withheld);
2940 }
2941 drop(coordination);
2942
2943 record_handover(
2944 &mut self.state,
2945 "implement",
2946 &seat.key,
2947 &seat.agent,
2948 &next.id,
2949 &cur,
2950 &fail_reason(&*out),
2951 );
2952 prev = Some(cur.clone());
2953
2954 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2955 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2956 job.spec = next.clone();
2964 let mut retry = job.clone();
2965 retry.seat = new_seat;
2966 retry.prompt = prompt::implement_in(
2967 &instruction,
2968 &job.cwd.to_string_lossy(),
2969 &language,
2970 &github_language,
2971 brief.as_deref(),
2972 &attachments,
2973 );
2974 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2975 let cache = self.state.config.cache_dir();
2976 let ctx = WaveCtx {
2977 carry_seats: false,
2978 pauses: Some((&self.pause, &self.interrupt)),
2979 run: run_id,
2980 node: "implement",
2981 prompts,
2982 cache: cache.as_deref(),
2983 round: None,
2984 };
2985 let (fallback_seat, fallback_out) = run_one(
2986 retry,
2987 Arc::clone(&self.sem),
2988 &ctx,
2989 &mut self.state,
2990 fallback_attempt,
2991 )
2992 .await;
2993 *seat = fallback_seat;
2994 *out = fallback_out;
2995 }
2996 }
2997 Ok(())
2998 }
2999
3000 async fn resume_unconfirmed_commands(
3024 &mut self,
3025 results: &mut [(usize, SeatState, AgentOutcome)],
3026 sent: &[SeatJob],
3027 prompts: &Prompts,
3028 run_id: &str,
3029 ) {
3030 for (wi, seat, out) in results.iter_mut() {
3031 let AgentOutcome::Ok(o) = &*out else {
3032 continue;
3033 };
3034 if !has_unconfirmed_command(&o.commands) {
3035 continue;
3036 }
3037 let Some(job) = sent.get(*wi) else { continue };
3038 if !has_context(&job.spec, seat, job.sessions) {
3039 self.state.event(
3040 "implement",
3041 format!(
3042 "{}: the reply named a command whose own CLI never confirmed the exit \
3043 status of, but there is no session left to resume",
3044 seat.key
3045 ),
3046 );
3047 continue;
3048 }
3049 self.state.event(
3050 "implement",
3051 format!(
3052 "{}: the reply named a command whose own CLI never confirmed the exit \
3053 status of; resuming the conversation",
3054 seat.key
3055 ),
3056 );
3057 let mut retry = job.clone();
3058 retry.seat = seat.clone();
3059 retry.prompt = prompt::resume_incomplete(
3060 "a command in your last reply had no confirmed exit status",
3061 );
3062 retry.timeout = retry_budget(job.timeout, true);
3063 retry.stem = format!("{}-confirm", job.stem);
3064 let cache = self.state.config.cache_dir();
3065 let ctx = WaveCtx {
3066 carry_seats: false,
3067 pauses: Some((&self.pause, &self.interrupt)),
3068 run: run_id,
3069 node: "implement",
3070 prompts,
3071 cache: cache.as_deref(),
3072 round: None,
3073 };
3074 let (resumed_seat, resumed) =
3075 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3076 *seat = resumed_seat;
3077 *out = resumed;
3078 }
3079 }
3080
3081 async fn continue_fix_report(
3102 &mut self,
3103 mut seat: SeatState,
3104 parse_err: String,
3105 job: &SeatJob,
3106 prompts: &Prompts,
3107 run_id: &str,
3108 round: usize,
3109 ) -> (
3110 SeatState,
3111 Option<FixReport>,
3112 Option<String>,
3113 ContinuationRecord,
3114 ) {
3115 let mut last_err = parse_err;
3116 let mut cumulative_wait_ms = 0u64;
3117 let mut attempts = 0usize;
3118 loop {
3119 if !has_context(&job.spec, &seat, job.sessions) {
3120 self.state.event(
3121 "fix",
3122 format!(
3123 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3124 session left to resume into"
3125 ),
3126 );
3127 let outcome = if attempts == 0 {
3128 ContinuationOutcome::NoSession
3129 } else {
3130 ContinuationOutcome::Exhausted
3131 };
3132 return (
3133 seat,
3134 None,
3135 Some(format!("unparsable fix report: {last_err}")),
3136 ContinuationRecord {
3137 attempts,
3138 cumulative_wait_ms,
3139 outcome,
3140 },
3141 );
3142 }
3143 if attempts >= MAX_FIX_CONTINUATIONS {
3144 self.state.event(
3145 "fix",
3146 format!(
3147 "round {round}: fixer's reply still had no adoption report after \
3148 {attempts} continuation(s) ({last_err}); giving up"
3149 ),
3150 );
3151 return (
3152 seat,
3153 None,
3154 Some(format!(
3155 "unparsable fix report after {attempts} continuation(s): {last_err}"
3156 )),
3157 ContinuationRecord {
3158 attempts,
3159 cumulative_wait_ms,
3160 outcome: ContinuationOutcome::Exhausted,
3161 },
3162 );
3163 }
3164 attempts += 1;
3165 self.state.event(
3166 "fix",
3167 format!(
3168 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3169 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3170 ),
3171 );
3172 let mut retry = job.clone();
3173 retry.seat = seat.clone();
3174 retry.prompt = prompt::resume_incomplete(&last_err);
3175 retry.timeout = retry_budget(job.timeout, true);
3176 retry.stem = format!("{}-continue{attempts}", job.stem);
3177 let cache = self.state.config.cache_dir();
3178 let ctx = WaveCtx {
3179 carry_seats: false,
3180 pauses: Some((&self.pause, &self.interrupt)),
3181 run: run_id,
3182 node: "fix",
3183 prompts,
3184 cache: cache.as_deref(),
3185 round: Some(round),
3186 };
3187 let (resumed_seat, resumed_out) = run_one(
3188 retry,
3189 Arc::clone(&self.sem),
3190 &ctx,
3191 &mut self.state,
3192 attempts,
3193 )
3194 .await;
3195 seat = resumed_seat;
3196 match resumed_out {
3197 AgentOutcome::Ok(o) => {
3198 cumulative_wait_ms += o.duration_ms;
3199 match verdict::extract_json::<FixReport>(&o.text) {
3200 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3201 self.state.event(
3202 "fix",
3203 format!(
3204 "round {round}: fixer's adoption report recovered after \
3205 {attempts} continuation(s)"
3206 ),
3207 );
3208 return (
3209 seat,
3210 Some(report),
3211 None,
3212 ContinuationRecord {
3213 attempts,
3214 cumulative_wait_ms,
3215 outcome: ContinuationOutcome::Resumed,
3216 },
3217 );
3218 }
3219 Ok(_) => {
3227 last_err = "the reply parsed, but it reported a command whose own CLI \
3228 never confirmed an exit status"
3229 .to_owned();
3230 }
3231 Err(e) => last_err = e.to_string(),
3232 }
3233 }
3234 AgentOutcome::Quota(o) => {
3235 cumulative_wait_ms += o.duration_ms;
3236 self.state.quota.push(QuotaLoss {
3237 seat: seat.key.clone(),
3238 node: "fix".to_owned(),
3239 at: Timestamp::now(),
3240 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3241 });
3242 self.state.event(
3243 "fix",
3244 format!(
3245 "round {round}: continuation rate limited (quota); not retrying now"
3246 ),
3247 );
3248 return (
3249 seat,
3250 None,
3251 Some("rate limited (quota) while recovering the fix report".to_owned()),
3252 ContinuationRecord {
3253 attempts,
3254 cumulative_wait_ms,
3255 outcome: ContinuationOutcome::QuotaLost,
3256 },
3257 );
3258 }
3259 AgentOutcome::Dropped(o) => {
3260 cumulative_wait_ms += o.duration_ms;
3261 let why = o
3262 .dropped
3263 .as_ref()
3264 .map(|d| d.why.as_str())
3265 .unwrap_or("the CLI ended the stream without delivering its answer");
3266 last_err = format!("the CLI dropped the stream ({why})");
3267 }
3268 AgentOutcome::Failed(e) => last_err = e,
3269 AgentOutcome::ResourceBlocked(e) => {
3272 return (
3273 seat,
3274 None,
3275 Some(format!("the continuation was not started: {e}")),
3276 ContinuationRecord {
3277 attempts: attempts.saturating_sub(1),
3278 cumulative_wait_ms,
3279 outcome: ContinuationOutcome::NoSession,
3280 },
3281 );
3282 }
3283 }
3284 }
3285 }
3286
3287 fn after_implement(&mut self) -> Result<()> {
3288 if !self.state.candidate_preservation.is_empty() {
3289 bail!("candidate preservation incomplete; judging is unsafe");
3290 }
3291 if self.state.leaks.is_empty() {
3293 let cfg = self.state.config.blind.clone();
3294 let mut leaks = Vec::new();
3295 for c in &self.state.candidates {
3296 let Some(patch) =
3297 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3298 else {
3299 continue;
3300 };
3301 leaks.extend(blind::scan(
3302 &format!("candidate {} patch", c.label),
3303 &patch,
3304 &cfg.vendor_tokens,
3305 ));
3306 }
3307 if !leaks.is_empty() {
3308 let summary = leaks
3309 .iter()
3310 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3311 .collect::<Vec<_>>()
3312 .join(", ");
3313 match cfg.on_leak {
3314 LeakPolicy::Fail => {
3315 self.state.status = RunStatus::Failed;
3316 self.state
3317 .event("blind", format!("vendor text in a patch: {summary}"));
3318 self.state.leaks = leaks;
3319 self.state.save()?;
3320 self.settle_questions();
3321 bail!(
3322 "blind.on_leak = \"fail\" and vendor text reached a \
3323 judged patch: {summary}"
3324 );
3325 }
3326 LeakPolicy::Redact => self.state.event(
3327 "blind",
3328 format!("redacting vendor text for judging: {summary}"),
3329 ),
3330 LeakPolicy::Warn => self.state.event(
3331 "blind",
3332 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3333 ),
3334 }
3335 self.state.leaks = leaks;
3336 }
3337 }
3338
3339 if self.state.viable().is_empty() {
3340 if self.state.all_candidates_verified_noop() {
3341 self.state.status = RunStatus::VerifiedNoop;
3352 self.state.save()?;
3353 self.settle_questions();
3354 return Ok(());
3355 }
3356 self.state.status = RunStatus::Failed;
3357 self.state.save()?;
3358 self.settle_questions();
3359 bail!("no candidate produced a change; nothing to judge");
3360 }
3361 self.state.status = RunStatus::Judging;
3362 self.state.save()?;
3363 Ok(())
3364 }
3365
3366 async fn judge(&mut self) -> Result<()> {
3369 let run_id = self.state.id.clone();
3374 let prompts = self.state.config.prompts.clone();
3375 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3376 return Ok(());
3377 }
3378 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3379 if viable.len() == 1 {
3380 self.state.judge_skipped = true;
3387 self.state.event(
3388 "judge",
3389 format!(
3390 "only candidate {} produced a change; judging skipped",
3391 viable[0].label
3392 ),
3393 );
3394 self.state.save()?;
3395 return Ok(());
3396 }
3397 self.state.status = RunStatus::Judging;
3398
3399 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3400 let language = self.state.config.graph.language.clone();
3401 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3402 let sessions = self.state.config.graph.sessions;
3403 let artifacts = agent::artifacts_dir(&self.state.dir());
3404 let root = self.state.worktree_root();
3405 let base_short = short(&self.state.base_commit);
3406
3407 let mut jobs = Vec::new();
3408 let mut orders = Vec::new();
3409 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3410 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3411 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3412 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3413 let seat_key = format!("judge-{}", j + 1);
3414 let seat = self.seat(&seat_key, &spec.id);
3415 jobs.push(SeatJob {
3416 prompt: prompt::judge(
3417 &self.state.instruction,
3418 &views,
3419 self.roles.judges.len(),
3420 &base_short,
3421 &language,
3422 ),
3423 spec,
3424 seat,
3425 cwd: root.join(format!("judge-{}", j + 1)),
3426 timeout,
3427 allow_write: false,
3428 sessions,
3429 artifacts: artifacts.clone(),
3430 stem: format!("judge-{}", j + 1),
3431 handover: None,
3432 });
3433 }
3434
3435 self.state.event(
3436 "judge",
3437 format!(
3438 "{} judges ranking {} candidates blind",
3439 jobs.len(),
3440 viable.len()
3441 ),
3442 );
3443 let labels_for_check = labels.clone();
3444 let mut quota_losses = Vec::new();
3445 let cache = self.state.config.cache_dir();
3446 let ctx = WaveCtx {
3447 carry_seats: false,
3448 pauses: Some((&self.pause, &self.interrupt)),
3449 run: &run_id,
3450 node: "judge",
3451 prompts: &prompts,
3452 cache: cache.as_deref(),
3453 round: None,
3454 };
3455 let results = ask_json_wave::<Ranking>(
3456 jobs,
3457 Arc::clone(&self.sem),
3458 self.state.config.graph.retries,
3459 &self.roles.judge_roster,
3460 &ctx,
3461 &mut quota_losses,
3462 &mut self.state,
3463 &move |r: &Ranking| r.validate(&labels_for_check),
3464 )
3465 .await;
3466 if self.state.resource_block.is_some() {
3467 return self.stop_for_resource();
3468 }
3469 self.state.quota.extend(quota_losses);
3470
3471 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3472 let agent_id = seat.agent.clone();
3473 self.state.seats.insert(seat.key.clone(), seat);
3474 let mut record = Judgement {
3475 judge: j + 1,
3476 seat: format!("judge-{}", j + 1),
3477 agent: agent_id,
3478 ranking: Vec::new(),
3479 reasons: BTreeMap::new(),
3480 confidence: None,
3481 strengths: BTreeMap::new(),
3482 order: orders[j].clone(),
3483 failed: None,
3484 duration_ms: 0,
3485 };
3486 match res {
3487 Ok((ranking, out)) => {
3488 record.ranking = ranking.normalized();
3489 record.reasons = ranking.reasons;
3490 record.strengths = ranking.strengths;
3491 record.confidence = ranking.confidence;
3492 record.duration_ms = out.duration_ms;
3493 self.state.event(
3494 "judge",
3495 format!(
3496 "judge {} ranked {}",
3497 j + 1,
3498 record.ranking.iter().collect::<String>()
3499 ),
3500 );
3501 }
3502 Err(e) => {
3503 record.failed = Some(e.to_string());
3504 self.state
3505 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3506 }
3507 }
3508 self.state.judgements.push(record);
3509 self.state.save()?;
3510 }
3511 Ok(())
3512 }
3513
3514 async fn deliberate(&mut self) -> Result<()> {
3517 let run_id = self.state.id.clone();
3522 let prompts = self.state.config.prompts.clone();
3523 if !self.state.deliberation.is_empty() {
3524 return Ok(());
3525 }
3526 let tops: Vec<char> = self
3527 .state
3528 .judgements
3529 .iter()
3530 .filter_map(|j| j.ranking.first().copied())
3531 .collect();
3532 let rounds = self.state.config.graph.deliberate_rounds;
3533 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3534 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3535 self.state.event(
3536 "deliberate",
3537 format!("judges agreed on {} outright; no deliberation", tops[0]),
3538 );
3539 }
3540 self.state.status = RunStatus::Voting;
3541 self.state.save()?;
3542 return Ok(());
3543 }
3544
3545 self.state.status = RunStatus::Deliberating;
3546 self.state.event(
3547 "deliberate",
3548 format!(
3549 "split: first choices were {} — opening {rounds} round(s)",
3550 tops.iter().collect::<String>()
3551 ),
3552 );
3553
3554 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3555 let language = self.state.config.graph.language.clone();
3556 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3557 let sessions = self.state.config.graph.sessions;
3558 let artifacts = agent::artifacts_dir(&self.state.dir());
3559 let root = self.state.worktree_root();
3560 let base_short = short(&self.state.base_commit);
3561
3562 for round in 1..=rounds {
3566 let mut turns: Vec<DeliberationTurn> = Vec::new();
3567 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3568 if self.state.judgements[j].failed.is_some() {
3569 continue;
3570 }
3571 let seat_key = format!("judge-{}", j + 1);
3572 let spec = self.occupant(&seat_key, spec);
3573 let mut seat = self.seat(&seat_key, &spec.id);
3574 let transcript = self.transcript(&turns, j);
3575 let build = |context: Option<&str>| {
3576 prompt::deliberate(
3577 &self.state.instruction,
3578 context,
3579 &transcript,
3580 round,
3581 rounds,
3582 &language,
3583 )
3584 };
3585 let block = self.candidate_block(&viable, &base_short);
3586 let full = build(Some(&block));
3587 let text = if has_context(&spec, &seat, sessions) {
3588 build(None)
3589 } else {
3590 full.clone()
3591 };
3592 let job = SeatJob {
3593 spec,
3594 seat: seat.clone(),
3595 prompt: text,
3596 cwd: root.join(format!("judge-{}", j + 1)),
3597 timeout,
3598 allow_write: false,
3599 sessions,
3600 artifacts: artifacts.clone(),
3601 stem: format!("delib-{round}-judge-{}", j + 1),
3602 handover: Some(full),
3603 };
3604 let cache = self.state.config.cache_dir();
3605 let ctx = WaveCtx {
3606 carry_seats: false,
3607 pauses: Some((&self.pause, &self.interrupt)),
3608 run: &run_id,
3609 node: "deliberate",
3610 prompts: &prompts,
3611 cache: cache.as_deref(),
3612 round: None,
3613 };
3614 let mut losses = Vec::new();
3618 let mut results = ask_wave_with::<String>(
3619 vec![job],
3620 Arc::clone(&self.sem),
3621 0,
3622 &self.roles.judge_roster,
3623 &ctx,
3624 &mut losses,
3625 &mut self.state,
3626 &|text: &str| {
3627 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3628 },
3629 )
3630 .await;
3631 self.state.quota.extend(losses);
3632 let (updated, res, _) = results.pop().expect("one job in, one result out");
3633 seat = updated;
3634 let agent_id = seat.agent.clone();
3635 self.state.seats.insert(seat.key.clone(), seat);
3636 let body = match res {
3637 Ok((body, _)) => body,
3638 Err(e) => {
3641 self.state
3642 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3643 continue;
3644 }
3645 };
3646 let tentative = verdict::extract_json::<Position>(&body)
3647 .ok()
3648 .and_then(|p| p.tentative)
3649 .and_then(|s| s.trim().chars().next())
3650 .map(|c| c.to_ascii_uppercase());
3651 self.state.event(
3652 "deliberate",
3653 format!(
3654 "round {round}: judge {} now favours {}",
3655 j + 1,
3656 tentative.map_or("—".to_owned(), |c| c.to_string())
3657 ),
3658 );
3659 turns.push(DeliberationTurn {
3660 judge: j + 1,
3661 agent: agent_id,
3662 body: blind::sanitize_prose(&body, &self.state.config.blind),
3663 tentative,
3664 });
3665 }
3666 self.state
3667 .deliberation
3668 .push(DeliberationRound { round, turns });
3669 self.state.save()?;
3670 }
3671
3672 self.state.status = RunStatus::Voting;
3673 self.state.save()?;
3674 Ok(())
3675 }
3676
3677 async fn vote(&mut self) -> Result<()> {
3680 let run_id = self.state.id.clone();
3685 let prompts = self.state.config.prompts.clone();
3686 if !self.state.votes.is_empty() {
3687 return Ok(());
3688 }
3689 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3690 if viable.len() == 1 {
3691 return Ok(());
3692 }
3693 self.state.status = RunStatus::Voting;
3694
3695 let language = self.state.config.graph.language.clone();
3696 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3697 let sessions = self.state.config.graph.sessions;
3698 let artifacts = agent::artifacts_dir(&self.state.dir());
3699 let root = self.state.worktree_root();
3700 let base_short = short(&self.state.base_commit);
3701 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3702
3703 let mut jobs = Vec::new();
3704 let mut seats_at = Vec::new();
3705 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3706 if self
3707 .state
3708 .judgements
3709 .get(j)
3710 .is_some_and(|r| r.failed.is_some())
3711 {
3712 continue;
3713 }
3714 let seat_key = format!("judge-{}", j + 1);
3715 let spec = self.occupant(&seat_key, spec);
3716 let seat = self.seat(&seat_key, &spec.id);
3717 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3718 let text = if has_context(&spec, &seat, sessions) {
3719 prompt::final_vote(&viable, &language)
3720 } else {
3721 full.clone()
3722 };
3723 jobs.push(SeatJob {
3724 spec,
3725 seat,
3726 prompt: text,
3727 cwd: root.join(format!("judge-{}", j + 1)),
3728 timeout,
3729 allow_write: false,
3730 sessions,
3731 artifacts: artifacts.clone(),
3732 stem: format!("vote-judge-{}", j + 1),
3733 handover: Some(full),
3734 });
3735 seats_at.push(j);
3736 }
3737
3738 self.state.event(
3739 "vote",
3740 format!(
3741 "collecting {} final votes one by one, privately",
3742 jobs.len()
3743 ),
3744 );
3745 let allowed = viable.clone();
3746 let mut quota_losses = Vec::new();
3747 let cache = self.state.config.cache_dir();
3748 let ctx = WaveCtx {
3749 carry_seats: false,
3750 pauses: Some((&self.pause, &self.interrupt)),
3751 run: &run_id,
3752 node: "vote",
3753 prompts: &prompts,
3754 cache: cache.as_deref(),
3755 round: None,
3756 };
3757 let results = ask_json_wave::<FinalVote>(
3758 jobs,
3759 Arc::clone(&self.sem),
3760 self.state.config.graph.retries,
3761 &self.roles.judge_roster,
3762 &ctx,
3763 &mut quota_losses,
3764 &mut self.state,
3765 &move |v: &FinalVote| match v.label() {
3766 Some(c) if allowed.contains(&c) => Ok(()),
3767 other => bail!("vote {other:?} is not one of {allowed:?}"),
3768 },
3769 )
3770 .await;
3771 if self.state.resource_block.is_some() {
3772 return self.stop_for_resource();
3773 }
3774 self.state.quota.extend(quota_losses);
3775
3776 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3777 let agent_id = seat.agent.clone();
3778 self.state.seats.insert(seat.key.clone(), seat);
3779 let initial = self
3780 .state
3781 .judgements
3782 .get(j)
3783 .and_then(|r| r.ranking.first().copied());
3784 let mut record = VoteRecord {
3785 judge: j + 1,
3786 agent: agent_id,
3787 vote: None,
3788 reason: String::new(),
3789 changed: false,
3790 };
3791 match res {
3792 Ok((v, _)) => {
3793 record.vote = v.label();
3794 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3795 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3796 self.state.event(
3797 "vote",
3798 format!(
3799 "judge {} voted {}{}",
3800 j + 1,
3801 record.vote.unwrap_or('?'),
3802 if record.changed { " (changed)" } else { "" }
3803 ),
3804 );
3805 }
3806 Err(e) => {
3807 self.state
3808 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3809 }
3810 }
3811 self.state.votes.push(record);
3812 self.state.save()?;
3813 }
3814 Ok(())
3815 }
3816
3817 fn tally(&mut self) -> Result<()> {
3820 if self.state.tally.is_some() {
3821 return Ok(());
3822 }
3823 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3824 let tops: Vec<char> = self
3825 .state
3826 .judgements
3827 .iter()
3828 .filter_map(|j| j.ranking.first().copied())
3829 .collect();
3830 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3831
3832 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3835 let mut cast: Vec<char> = Vec::new();
3836 for (i, j) in self.state.judgements.iter().enumerate() {
3837 let vote = self
3838 .state
3839 .votes
3840 .iter()
3841 .find(|v| v.judge == i + 1)
3842 .and_then(|v| v.vote)
3843 .or_else(|| j.ranking.first().copied());
3844 if let Some(v) = vote {
3845 *first_choice.entry(v).or_insert(0) += 1;
3846 cast.push(v);
3847 }
3848 }
3849
3850 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3851 for j in &self.state.judgements {
3852 let n = j.ranking.len();
3853 for (pos, label) in j.ranking.iter().enumerate() {
3854 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3855 }
3856 }
3857
3858 let best = first_choice.values().copied().max().unwrap_or(0);
3859 let mut leaders: Vec<char> = first_choice
3860 .iter()
3861 .filter(|(_, v)| **v == best)
3862 .map(|(k, _)| *k)
3863 .collect();
3864 let mut tie_break = None;
3865 if leaders.len() > 1 {
3866 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3867 let borda_leaders: Vec<char> = leaders
3868 .iter()
3869 .copied()
3870 .filter(|l| borda[l] == top_borda)
3871 .collect();
3872 tie_break = Some(if borda_leaders.len() == 1 {
3873 format!(
3874 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3875 leaders.len()
3876 )
3877 } else {
3878 format!(
3879 "{} way tie on both first-choice votes and Borda points, broken by label order",
3880 leaders.len()
3881 )
3882 });
3883 leaders = borda_leaders;
3884 leaders.sort_unstable();
3885 }
3886 let winner = *leaders
3887 .first()
3888 .or(viable.first())
3889 .context("no candidate to declare a winner from")?;
3890
3891 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3892 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3893 let deliberated = !self.state.deliberation.is_empty();
3894
3895 let quota_seats: std::collections::BTreeSet<&str> =
3899 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3900 let mut present = 0usize;
3901 for (i, j) in self.state.judgements.iter().enumerate() {
3902 if quota_seats.contains(j.seat.as_str()) {
3903 continue;
3904 }
3905 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3906 let voted = self
3907 .state
3908 .votes
3909 .iter()
3910 .any(|v| v.judge == i + 1 && v.vote.is_some());
3911 if ranked || voted {
3912 present += 1;
3913 }
3914 }
3915 let needs_quorum = viable.len() > 1;
3921 let judges_total = if needs_quorum {
3922 self.roles.judges.len()
3923 } else {
3924 0
3925 };
3926 let quorum = if needs_quorum {
3927 judges_total / 2 + 1
3928 } else {
3929 0
3930 };
3931 let met_quorum = !needs_quorum || present >= quorum;
3932 let uncontested = (!needs_quorum).then(|| {
3933 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3934 });
3935
3936 self.state.event(
3937 "tally",
3938 match &uncontested {
3939 Some(reason) => format!("winner {winner} — {reason}"),
3940 None => format!(
3941 "winner {winner} — votes {} | initial {} | {} changed | \
3942 {present}/{judges_total} judges{}",
3943 first_choice
3944 .iter()
3945 .map(|(k, v)| format!("{k}:{v}"))
3946 .collect::<Vec<_>>()
3947 .join(" "),
3948 if unanimous_initial {
3949 "unanimous"
3950 } else {
3951 "split"
3952 },
3953 changed_votes,
3954 if met_quorum {
3955 String::new()
3956 } else {
3957 format!(" — below quorum ({quorum} required)")
3958 },
3959 ),
3960 },
3961 );
3962 if !met_quorum {
3963 self.state.event(
3964 "stall",
3965 format!(
3966 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3967 the run stops here, resumable"
3968 ),
3969 );
3970 }
3971 self.state.tally = Some(Tally {
3972 first_choice,
3973 borda,
3974 winner,
3975 rankings: tops.len(),
3976 unanimous_initial,
3977 deliberated,
3978 changed_votes,
3979 unanimous_final,
3980 tie_break,
3981 judges: judges_total,
3982 present,
3983 quorum,
3984 met_quorum,
3985 uncontested,
3986 });
3987 self.state.status = if met_quorum {
3988 RunStatus::Reviewing
3989 } else {
3990 RunStatus::Stalled
3991 };
3992 self.state.save()?;
3993 Ok(())
3994 }
3995
3996 #[allow(clippy::too_many_lines)]
4017 async fn recover_stall(&mut self) -> Result<bool> {
4018 let run_id = self.state.id.clone();
4023 let prompts = self.state.config.prompts.clone();
4024 let quota_seats: BTreeSet<&str> =
4029 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4030 let absent: Vec<String> = self
4031 .state
4032 .judgements
4033 .iter()
4034 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4035 .map(|j| j.seat.clone())
4036 .collect();
4037 if absent.is_empty() {
4038 return Ok(false);
4039 }
4040 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4041 if viable.len() <= 1 {
4042 return Ok(false);
4043 }
4044 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4045 let language = self.state.config.graph.language.clone();
4046 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4047 let sessions = self.state.config.graph.sessions;
4048 let artifacts = agent::artifacts_dir(&self.state.dir());
4049 let root = self.state.worktree_root();
4050 let base_short = short(&self.state.base_commit);
4051 let candidates: Vec<Candidate> = viable.clone();
4052
4053 let mut positions: Vec<usize> = absent
4055 .iter()
4056 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4057 .collect();
4058 if positions.is_empty() {
4059 return Ok(false);
4060 }
4061 positions.sort_unstable();
4062 positions.dedup();
4063
4064 let mut judge_jobs = Vec::new();
4066 for &j in &positions {
4067 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4068 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4069 let seat_key = format!("judge-{}", j + 1);
4070 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4071 let seat = self.seat(&seat_key, &spec.id);
4072 judge_jobs.push(SeatJob {
4073 spec,
4074 seat,
4075 prompt: prompt::judge(
4076 &self.state.instruction,
4077 &views,
4078 self.roles.judges.len(),
4079 &base_short,
4080 &language,
4081 ),
4082 cwd: root.join(seat_key),
4083 timeout,
4084 allow_write: false,
4085 sessions,
4086 artifacts: artifacts.clone(),
4087 stem: format!("judge-{}-recover", j + 1),
4088 handover: None,
4089 });
4090 }
4091
4092 let labels_for_check = labels.clone();
4093 let mut judge_losses = Vec::new();
4094 let retries = self.state.config.graph.retries;
4095 let cache = self.state.config.cache_dir();
4096 let ctx = WaveCtx {
4097 carry_seats: false,
4098 pauses: Some((&self.pause, &self.interrupt)),
4099 run: &run_id,
4100 node: "judge",
4101 prompts: &prompts,
4102 cache: cache.as_deref(),
4103 round: None,
4104 };
4105 let results = ask_json_wave::<Ranking>(
4106 judge_jobs,
4107 Arc::clone(&self.sem),
4108 retries,
4109 &self.roles.judge_roster,
4110 &ctx,
4111 &mut judge_losses,
4112 &mut self.state,
4113 &move |r: &Ranking| r.validate(&labels_for_check),
4114 )
4115 .await;
4116 if self.state.resource_block.is_some() {
4117 self.stop_for_resource()?;
4118 return Ok(false);
4119 }
4120
4121 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4123 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4124 let agent_id = seat.agent.clone();
4125 self.state.seats.insert(seat.key.clone(), seat);
4126 let record = &mut self.state.judgements[j];
4127 match res {
4128 Ok((ranking, out)) => {
4129 record.agent = agent_id;
4130 record.ranking = ranking.normalized();
4131 record.reasons = ranking.reasons;
4132 record.strengths = ranking.strengths;
4133 record.confidence = ranking.confidence;
4134 record.failed = None;
4135 record.duration_ms = out.duration_ms;
4136 recovered.insert(j);
4137 self.state.event(
4138 "recover",
4139 format!("judge {} ranked again after the limit", j + 1),
4140 );
4141 }
4142 Err(e) => {
4143 self.state
4144 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4145 }
4146 }
4147 }
4148
4149 let mut vote_jobs = Vec::new();
4151 let mut vote_pos: Vec<usize> = Vec::new();
4152 for &j in &recovered {
4153 let seat_key = format!("judge-{}", j + 1);
4154 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4155 let seat = self.seat(&seat_key, &spec.id);
4156 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4157 let text = if has_context(&spec, &seat, sessions) {
4158 prompt::final_vote(&labels, &language)
4159 } else {
4160 full.clone()
4161 };
4162 vote_jobs.push(SeatJob {
4163 spec,
4164 seat,
4165 prompt: text,
4166 cwd: root.join(seat_key),
4167 timeout,
4168 allow_write: false,
4169 sessions,
4170 artifacts: artifacts.clone(),
4171 stem: format!("vote-judge-{}-recover", j + 1),
4172 handover: Some(full),
4173 });
4174 vote_pos.push(j);
4175 }
4176 let allowed = labels.clone();
4177 let mut vote_losses = Vec::new();
4178 let vote_retries = self.state.config.graph.retries;
4179 let vote_cache = self.state.config.cache_dir();
4180 let ctx = WaveCtx {
4181 carry_seats: false,
4182 pauses: Some((&self.pause, &self.interrupt)),
4183 run: &run_id,
4184 node: "vote",
4185 prompts: &prompts,
4186 cache: vote_cache.as_deref(),
4187 round: None,
4188 };
4189 let votes = ask_json_wave::<FinalVote>(
4190 vote_jobs,
4191 Arc::clone(&self.sem),
4192 vote_retries,
4193 &self.roles.judge_roster,
4194 &ctx,
4195 &mut vote_losses,
4196 &mut self.state,
4197 &move |v: &FinalVote| match v.label() {
4198 Some(c) if allowed.contains(&c) => Ok(()),
4199 other => bail!("vote {other:?} is not one of {allowed:?}"),
4200 },
4201 )
4202 .await;
4203 if self.state.resource_block.is_some() {
4204 self.stop_for_resource()?;
4205 return Ok(false);
4206 }
4207 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4208 let agent_id = seat.agent.clone();
4209 self.state.seats.insert(seat.key.clone(), seat);
4210 match res {
4211 Ok((v, _)) => {
4212 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4213 rec.vote = v.label();
4214 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4215 } else {
4216 self.state.votes.push(VoteRecord {
4217 judge: j + 1,
4218 agent: agent_id,
4219 vote: v.label(),
4220 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4221 changed: false,
4222 });
4223 }
4224 self.state.event(
4225 "recover",
4226 format!("judge {} voted again after the limit", j + 1),
4227 );
4228 }
4229 Err(e) => {
4230 self.state
4231 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4232 }
4233 }
4234 }
4235
4236 let recovered_keys: BTreeSet<String> = recovered
4240 .iter()
4241 .map(|&j| format!("judge-{}", j + 1))
4242 .collect();
4243 self.state
4244 .quota
4245 .retain(|q| !recovered_keys.contains(&q.seat));
4246 for loss in judge_losses.into_iter().chain(vote_losses) {
4250 if recovered_keys.contains(&loss.seat) {
4251 continue;
4252 }
4253 self.state.quota.retain(|q| q.seat != loss.seat);
4254 self.state.quota.push(loss);
4255 }
4256
4257 self.state.tally = None;
4259 self.tally()?;
4260 Ok(self
4261 .state
4262 .tally
4263 .as_ref()
4264 .map(|t| t.met_quorum)
4265 .unwrap_or(false))
4266 }
4267
4268 async fn graft(&mut self) -> Result<()> {
4284 let Err(e) = self.graft_inner().await else {
4285 return Ok(());
4286 };
4287 if e.is::<crate::repo_lock::ResourceBlocked>() {
4288 return Err(e);
4289 }
4290 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4291 self.state.status = RunStatus::Blocked;
4292 self.state.event("graft", format!("blocked: {e:#}"));
4293 self.state.save()?;
4294 return Err(e);
4295 }
4296 if let Some(rec) = self.state.graft.clone()
4301 && !rec.outcome.settled()
4302 {
4303 if let Some(winner) = self.state.winner().cloned() {
4304 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4305 if re.is::<crate::repo_lock::ResourceBlocked>() {
4306 return Err(re);
4307 }
4308 self.state.status = RunStatus::Blocked;
4309 self.state.event("graft", format!("blocked: {re:#}"));
4310 self.state.save()?;
4311 return Err(re);
4312 }
4313 }
4314 if let Some(g) = self.state.graft.as_mut() {
4315 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4316 }
4317 }
4318 self.state.event("graft", format!("skipped: {e:#}"));
4319 if let Err(e) = self.state.save() {
4320 tracing::warn!("save after graft: {e:#}");
4321 }
4322 Ok(())
4323 }
4324
4325 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4328 let _coordination = self.coordinate(wt).await?;
4329 crate::recovery::restore_repair(wt, pre)
4330 .await
4331 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4332 }
4333
4334 fn graft_skipped(&mut self, why: &str) {
4336 let message = format!("skipped: {why}");
4337 if !self
4338 .state
4339 .events
4340 .iter()
4341 .any(|e| e.node == "graft" && e.message == message)
4342 {
4343 self.state.event("graft", message);
4344 }
4345 }
4346
4347 async fn graft_inner(&mut self) -> Result<()> {
4348 if let Some(rec) = self.state.graft.clone()
4351 && !rec.outcome.settled()
4352 {
4353 if self.state.reviews.is_empty()
4357 && let Some(winner) = self.state.winner().cloned()
4358 {
4359 self.graft_restore(&winner.worktree, &rec.pre).await?;
4360 }
4361 if rec.outcome == GraftOutcome::WaitingForRestore {
4362 self.state.graft = None;
4363 self.state.save()?;
4364 } else {
4365 if let Some(g) = self.state.graft.as_mut() {
4366 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4367 }
4368 self.state.event(
4369 "graft",
4370 "rolled back: the previous round was interrupted; the winner is untouched",
4371 );
4372 self.state.save()?;
4373 return Ok(());
4374 }
4375 }
4376 let plan = match graft::plan(&self.state) {
4377 Ok(plan) => plan,
4378 Err(skip) => {
4379 if skip != graft::Skip::AlreadyDone {
4380 self.graft_skipped(skip.reason());
4381 self.state.save()?;
4382 }
4383 return Ok(());
4384 }
4385 };
4386 let Some(winner) = self.state.winner().cloned() else {
4387 return Ok(());
4388 };
4389 let wt = winner.worktree.clone();
4390 if !git::is_clean(&wt).await? {
4391 self.graft_skipped("the winner's worktree has uncommitted changes");
4392 self.state.save()?;
4393 return Ok(());
4394 }
4395 let pre = git::rev_parse(&wt, "HEAD").await?;
4396 self.state.graft = Some(GraftRecord {
4399 pre: pre.clone(),
4400 post: None,
4401 lines: None,
4402 limit: None,
4403 agent: None,
4404 outcome: GraftOutcome::Running,
4405 requested: graft::requested(&plan),
4406 adoption: None,
4407 adoption_error: None,
4408 });
4409 self.state.event(
4410 "graft",
4411 format!(
4412 "porting strengths of candidate(s) {} onto {}",
4413 plan.strengths
4414 .iter()
4415 .map(|(l, _, _)| l.to_string())
4416 .collect::<Vec<_>>()
4417 .join(", "),
4418 plan.winner
4419 ),
4420 );
4421 self.state.save()?;
4422
4423 let fix_prompt = prompt::graft(
4424 &self.state.instruction,
4425 plan.winner,
4426 &plan.strengths,
4427 &self.state.config.graph.language,
4428 );
4429 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4430 let sessions = self.state.config.graph.sessions;
4431 let artifacts = agent::artifacts_dir(&self.state.dir());
4432 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4433 let (_, seat, out) = self
4434 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4435 SeatJob {
4436 prompt: fix_prompt.clone(),
4437 spec,
4438 seat,
4439 cwd: wt.clone(),
4440 timeout,
4441 allow_write: true,
4442 sessions,
4443 artifacts: artifacts.clone(),
4444 stem: "graft".to_owned(),
4445 handover: None,
4446 }
4447 })
4448 .await;
4449 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4450 self.state.graft.as_mut().expect("graft record").outcome =
4453 GraftOutcome::WaitingForRestore;
4454 return self.stop_for_resource();
4455 }
4456 let mut why: Option<String> = None;
4457 match &out {
4458 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4459 why = Some("the fixer produced nothing usable".to_owned());
4460 }
4461 AgentOutcome::Ok(_) => {}
4462 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4463 AgentOutcome::Quota(o) => {
4464 self.state.quota.push(QuotaLoss {
4465 seat: seat.key.clone(),
4466 node: "graft".to_owned(),
4467 at: Timestamp::now(),
4468 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4469 });
4470 why = Some("rate limited (quota)".to_owned());
4471 }
4472 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4473 why = Some(format!(
4474 "the fixer failed: {}",
4475 e.lines()
4476 .next()
4477 .unwrap_or("")
4478 .chars()
4479 .take(160)
4480 .collect::<String>()
4481 ));
4482 }
4483 }
4484 let agent_id = seat.agent.clone();
4485 self.state.seats.insert(seat.key.clone(), seat);
4486
4487 let mut post = None;
4488 let mut lines = None;
4489 let _coordination = self.coordinate(&wt).await?;
4490 if why.is_none() {
4491 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4492 if let Ok(r) = &rescued {
4493 self.state.note_withheld("graft", &r.withheld);
4494 }
4495 let head = git::rev_parse(&wt, "HEAD").await?;
4496 let dirty = match &rescued {
4500 Ok(r) => git::status_porcelain(&wt)
4501 .await
4502 .map(|st| {
4503 st.lines().any(|l| {
4504 !r.withheld
4505 .iter()
4506 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4507 })
4508 })
4509 .unwrap_or(true),
4510 Err(_) => true,
4511 };
4512 if dirty {
4513 why = Some("the fixer left changes that could not be committed".to_owned());
4514 } else if head == pre {
4515 why = Some("the fixer changed nothing".to_owned());
4516 } else {
4517 lines = git::changed_lines(&wt, &pre, &head).await;
4518 post = Some(head);
4519 }
4520 }
4521 let (adoption, adoption_error) = match &out {
4522 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4523 _ => (None, None),
4524 };
4525 let requested = graft::requested(&plan);
4526 let outcome = match why {
4527 None => {
4528 self.state.event(
4529 "graft",
4530 format!(
4531 "applied: {}",
4532 lines.map_or_else(
4533 || "changed lines unavailable".to_owned(),
4534 |n| format!("{n} changed lines")
4535 )
4536 ),
4537 );
4538 GraftOutcome::Applied
4539 }
4540 Some(why) => {
4541 crate::recovery::restore_repair(&wt, &pre)
4544 .await
4545 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4546 self.state.event(
4547 "graft",
4548 format!("rolled back: {why}; the winner is untouched"),
4549 );
4550 GraftOutcome::RolledBack(why)
4551 }
4552 };
4553 self.state.graft = Some(GraftRecord {
4554 pre,
4555 post,
4556 lines,
4557 limit: None,
4558 agent: Some(agent_id),
4559 outcome,
4560 requested,
4561 adoption,
4562 adoption_error,
4563 });
4564 self.state.save()?;
4565 Ok(())
4566 }
4567
4568 async fn fold_losers(&mut self) -> Result<()> {
4571 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4572 return Ok(());
4573 };
4574 if !self.state.candidate_preservation.is_empty() {
4575 bail!("candidate preservation incomplete; refusing cleanup");
4576 }
4577 let repo = self.state.repo.clone();
4578 let _coordination = self.coordinate(&repo).await?;
4579 let mut folded = Vec::new();
4580 for i in 0..self.state.candidates.len() {
4581 let c = &self.state.candidates[i];
4582 if c.label == winner || c.folded {
4583 continue;
4584 }
4585 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4586 remove_preserved_worktree(&repo, &wt).await?;
4587 if git::branch_exists(&repo, &branch).await?
4588 && !git::branch_delete(&repo, &branch).await?
4589 {
4590 bail!("candidate branch removal refused");
4591 }
4592 self.state.candidates[i].folded = true;
4593 self.state.save()?;
4594 folded.push(label.to_string());
4595 }
4596 let root = self.state.worktree_root();
4598 for j in 1..=self.roles.judges.len() {
4599 let wt = root.join(format!("judge-{j}"));
4600 if wt.exists() {
4601 remove_preserved_worktree(&repo, &wt).await?;
4602 }
4603 }
4604 if self.state.config.graph.advise {
4607 for k in 1..=self.state.config.graph.advisors {
4608 let wt = root.join(format!("advisor-{k}"));
4609 if wt.exists() {
4610 remove_preserved_worktree(&repo, &wt).await?;
4611 }
4612 }
4613 }
4614 if !folded.is_empty() {
4615 self.state
4616 .event("fold", format!("folded candidates {}", folded.join(", ")));
4617 self.state.save()?;
4618 }
4619 Ok(())
4620 }
4621
4622 async fn sync_to_base(&mut self) -> Result<()> {
4662 if self.state.status == RunStatus::AlreadyInBase {
4663 return Ok(());
4664 }
4665 let conflicted = self
4666 .state
4667 .base_sync
4668 .as_ref()
4669 .is_some_and(|s| s.conflict.is_some());
4670 let Some(winner) = self.state.winner().cloned() else {
4671 return Ok(());
4672 };
4673
4674 let repo = self.state.repo.clone();
4675 let remote = self.state.config.merge.remote.clone();
4676 let base_branch = self.state.base_branch.clone();
4677 let tracking = format!("{remote}/{base_branch}");
4678
4679 git::fetch(&repo, &remote, &base_branch).await.ok();
4680 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4685 return Ok(());
4686 };
4687
4688 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4689 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4690 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4691
4692 if (behind > 0 || conflicted || head == tip)
4701 && self
4702 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4703 .await?
4704 {
4705 return Ok(());
4706 }
4707 if conflicted {
4708 return Ok(());
4709 }
4710
4711 if behind == 0 {
4712 if let Some(from) = self
4721 .state
4722 .rebase_fixes
4723 .iter()
4724 .rev()
4725 .find_map(|r| r.from.clone())
4726 && from != head
4727 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4728 .await
4729 .is_ok_and(|o| o.ok())
4730 {
4731 git::sync_to_head(&winner.worktree).await?;
4732 }
4733 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4739 if let Some(why) = &conflict {
4740 self.state.status = RunStatus::Blocked;
4741 self.state.event("land", why.clone());
4742 }
4743 self.state.base_sync = Some(BaseSync {
4744 tip,
4745 behind: 0,
4746 attempts,
4747 conflict,
4748 already_in: None,
4749 });
4750 self.state.save()?;
4751 return Ok(());
4752 }
4753
4754 if attempts >= BASE_SYNC_ROUNDS {
4755 let why = format!(
4756 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4757 rebase(s); rebasing again would only race it",
4758 winner.branch
4759 );
4760 self.state.status = RunStatus::Blocked;
4761 self.state.base_sync = Some(BaseSync {
4762 tip,
4763 behind,
4764 attempts,
4765 conflict: Some(why.clone()),
4766 already_in: None,
4767 });
4768 self.state.event("land", why);
4769 self.state.save()?;
4770 return Ok(());
4771 }
4772
4773 self.state.event(
4774 "land",
4775 format!(
4776 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4777 winner.branch
4778 ),
4779 );
4780 self.state.save()?;
4781
4782 let branch_tracking = format!("{remote}/{}", winner.branch);
4788 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4789 let remote_tip = if self.state.task_pr_branch().is_none()
4790 && matches!(&fetched_branch, Ok(o) if o.ok())
4791 {
4792 git::rev_parse(&repo, &branch_tracking).await.ok()
4793 } else {
4794 None
4795 };
4796 if let Some(theirs) = &remote_tip
4800 && !git::is_ancestor(&repo, theirs, &head).await
4801 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4802 .await
4803 .is_ok_and(|missing| missing.is_empty())
4804 {
4805 let why = format!(
4806 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4807 them",
4808 short(theirs),
4809 winner.branch
4810 );
4811 self.state.status = RunStatus::Blocked;
4812 self.state.base_sync = Some(BaseSync {
4813 tip,
4814 behind,
4815 attempts,
4816 conflict: Some(why.clone()),
4817 already_in: None,
4818 });
4819 self.state.event("land", why);
4820 self.state.save()?;
4821 return Ok(());
4822 }
4823
4824 let scratch = self.state.dir().join("base-sync");
4825 let rebased = match crate::rebase::rebase_with_fixer(
4826 &mut self.state,
4827 &scratch,
4828 &winner.branch,
4829 &tracking,
4830 )
4831 .await
4832 {
4833 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4834 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4835 Err(e) => Err(e),
4836 };
4837 let attempts = attempts + 1;
4838 match rebased {
4839 Ok(None) => {
4840 git::sync_to_head(&winner.worktree).await?;
4844 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4845 let mut conflict = None;
4846 if let Some(pinned) = &remote_tip {
4847 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4848 match pushed {
4849 Ok(o) if o.ok() => self.state.event(
4850 "land",
4851 format!("pushed rebased {} to {remote}", winner.branch),
4852 ),
4853 Ok(o) => {
4854 conflict = Some(format!(
4855 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4856 winner.branch,
4857 short(pinned),
4858 o.stderr.chars().take(600).collect::<String>()
4859 ));
4860 }
4861 Err(e) => {
4862 conflict = Some(format!(
4863 "rebased {} locally but could not push it: {e:#}",
4864 winner.branch
4865 ));
4866 }
4867 }
4868 }
4869 if let Some(why) = &conflict {
4870 self.state.status = RunStatus::Blocked;
4871 self.state.event("land", why.clone());
4872 }
4873 self.state.base_sync = Some(BaseSync {
4874 tip: tip.clone(),
4875 behind: 0,
4876 attempts,
4877 conflict,
4878 already_in: None,
4879 });
4880 self.state
4881 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4882 }
4883 Ok(Some(conflict)) => {
4884 let why = format!(
4885 "{} conflicts with {tracking} and did not rebase: {}",
4886 winner.branch,
4887 conflict.chars().take(600).collect::<String>()
4888 );
4889 self.state.status = RunStatus::Blocked;
4890 self.state.base_sync = Some(BaseSync {
4891 tip,
4892 behind,
4893 attempts,
4894 conflict: Some(why.clone()),
4895 already_in: None,
4896 });
4897 self.state.event("land", why);
4898 }
4899 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4903 self.state.save()?;
4904 return Ok(());
4905 }
4906 Err(e) => {
4907 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4908 self.state.status = RunStatus::Blocked;
4909 self.state.base_sync = Some(BaseSync {
4910 tip,
4911 behind,
4912 attempts,
4913 conflict: Some(why.clone()),
4914 already_in: None,
4915 });
4916 self.state.event("land", why);
4917 }
4918 }
4919 self.state.save()?;
4920 Ok(())
4921 }
4922
4923 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4929 if self.state.task_pr_branch().is_some() {
4932 return None;
4933 }
4934 let repo = self.state.repo.clone();
4935 let remote = self.state.config.merge.remote.clone();
4936 let fetched = git::fetch(&repo, &remote, branch).await;
4937 if !matches!(&fetched, Ok(o) if o.ok()) {
4938 return None;
4939 }
4940 let branch_tracking = format!("{remote}/{branch}");
4941 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4942 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4943 return None;
4944 }
4945 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4946 .await
4947 .is_ok_and(|missing| missing.is_empty())
4948 {
4949 return Some(format!(
4950 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4951 them",
4952 short(&theirs)
4953 ));
4954 }
4955 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4956 Ok(o) if o.ok() => {
4957 self.state
4958 .event("land", format!("pushed rebased {branch} to {remote}"));
4959 None
4960 }
4961 Ok(o) => Some(format!(
4962 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4963 someone may have pushed): {}",
4964 short(&theirs),
4965 o.stderr.chars().take(600).collect::<String>()
4966 )),
4967 Err(e) => Some(format!(
4968 "{branch} is rebased locally but could not be pushed: {e:#}"
4969 )),
4970 }
4971 }
4972
4973 async fn settle_already_in(
4990 &mut self,
4991 branch: &str,
4992 tip: &str,
4993 head: &str,
4994 attempts: usize,
4995 behind: usize,
4996 ) -> Result<bool> {
4997 let repo = self.state.repo.clone();
4998 let remote = self.state.config.merge.remote.clone();
4999 let start = self.state.base_commit.clone();
5000 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5001 Ok(Some(e)) => e,
5002 Ok(None) => return Ok(false),
5003 Err(e) => {
5004 tracing::warn!("already-in-base check for {branch}: {e:#}");
5005 return Ok(false);
5006 }
5007 };
5008 let mut verified = vec![head.to_owned()];
5009 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5010 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5011 if matches!(&fetched, Ok(o) if o.ok())
5012 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5013 && theirs != head
5014 {
5015 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5016 Ok(Some(_)) => verified.push(theirs),
5017 _ => return Ok(false),
5018 }
5019 }
5020 let base_branch = self.state.base_branch.clone();
5021 let message = format!(
5022 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5023 land",
5024 evidence.names(),
5025 evidence.proof.as_str()
5026 );
5027 let closed =
5028 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5029 .await;
5030 match closed {
5031 Ok(Ok(url)) => self
5032 .state
5033 .event("land", format!("closed {url}: superseded on {base_branch}")),
5034 Ok(Err(why)) => self
5035 .state
5036 .event("land", format!("did not close a pull request: {why}")),
5037 Err(e) => {
5038 let why = format!(
5039 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5040 not be closed ({e:#}); resume to retry"
5041 );
5042 self.state.status = RunStatus::Blocked;
5043 self.state.base_sync = Some(BaseSync {
5044 tip: tip.to_owned(),
5045 behind,
5046 attempts,
5047 conflict: Some(why.clone()),
5048 already_in: None,
5049 });
5050 self.state.event("land", why);
5051 self.state.save()?;
5052 return Ok(true);
5053 }
5054 }
5055 self.state.status = RunStatus::AlreadyInBase;
5056 self.state.base_sync = Some(BaseSync {
5057 tip: tip.to_owned(),
5058 behind,
5059 attempts,
5060 conflict: None,
5061 already_in: Some(evidence),
5062 });
5063 self.state.event("land", message);
5064 self.state.save()?;
5065 self.settle_questions();
5066 Ok(true)
5067 }
5068
5069 fn landing_base(&self) -> String {
5079 self.state
5080 .base_sync
5081 .as_ref()
5082 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5083 }
5084
5085 pub async fn fix_selected(
5118 &mut self,
5119 ids: &[String],
5120 reason: &str,
5121 allow_stale: bool,
5122 ) -> Result<()> {
5123 let reason = reason.trim();
5124 if reason.is_empty() {
5125 bail!("a fix request needs a reason — that is the operator's own record of why");
5126 }
5127 if ids.is_empty() {
5128 bail!("no finding id given");
5129 }
5130 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5131 bail!(
5132 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5133 has already concluded — can be given a targeted fix. A run still \
5134 in progress should simply be resumed; a `merged` run's branch has \
5135 already landed, so its answer is a fresh `magi review <branch>`, \
5136 not reopening this run's own record",
5137 self.state.id,
5138 self.state.status.as_str()
5139 );
5140 }
5141 let Some(winner) = self.state.winner().cloned() else {
5142 bail!("run {} has no winning candidate to fix", self.state.id);
5143 };
5144 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5145 bail!(
5146 "branch `{}` no longer exists; this run cannot be extended",
5147 winner.branch
5148 );
5149 }
5150 let home = crate::run::home();
5151 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5152 bail!(
5153 "run {} is currently being worked on by another magi process",
5154 self.state.id
5155 );
5156 }
5157 let _claim = FixClaim::acquire(&self.state.dir())?;
5163
5164 let mut seen = BTreeSet::new();
5168 let mut findings = Vec::new();
5169 let mut missing = Vec::new();
5170 for id in ids {
5171 if !seen.insert(id.clone()) {
5172 continue;
5173 }
5174 match self.state.finding(id) {
5175 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5176 id: f.id.clone(),
5177 severity: f.severity,
5178 reviewer_vote: rec.vote,
5179 round: round.round,
5180 round_head: round.head.clone(),
5181 reviewer: rec.reviewer,
5182 agent: rec.agent.clone(),
5183 file: f.file.clone(),
5184 line: f.line,
5185 title: f.title.clone(),
5186 detail: f.detail.clone(),
5187 outcome: OperatorFixOutcome::Pending,
5188 }),
5189 None => missing.push(id.clone()),
5190 }
5191 }
5192 if !missing.is_empty() {
5193 bail!(
5194 "unknown finding id(s): {}; nothing was changed",
5195 missing.join(", ")
5196 );
5197 }
5198
5199 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5200 let stale_details: Vec<(String, String)> = findings
5201 .iter()
5202 .filter(|f| f.round_head != head_at_request)
5203 .map(|f| (f.id.clone(), f.round_head.clone()))
5204 .collect();
5205 let stale = !stale_details.is_empty();
5206 if stale && !allow_stale {
5207 bail!(
5208 "the branch has moved since some finding(s) were raised — {} — now \
5209 at {}; pass --allow-stale to fix anyway, or re-run review first",
5210 stale_details
5211 .iter()
5212 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5213 .collect::<Vec<_>>()
5214 .join(", "),
5215 short(&head_at_request)
5216 );
5217 }
5218
5219 let request = OperatorFixRequest {
5220 requested_at: Timestamp::now(),
5221 reason: reason.to_owned(),
5222 findings,
5223 head_at_request: head_at_request.clone(),
5224 allow_stale,
5225 stale,
5226 fix: None,
5227 result_head: None,
5228 follow_up_review_run: None,
5229 };
5230 self.state.event(
5231 "fix",
5232 format!(
5233 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5234 request.findings.len(),
5235 request
5236 .findings
5237 .iter()
5238 .map(|f| f.id.as_str())
5239 .collect::<Vec<_>>()
5240 .join(", "),
5241 ),
5242 );
5243 self.state.operator_fixes.push(request);
5250 self.state.save()?;
5251 let request_index = self.state.operator_fixes.len() - 1;
5252
5253 let repo = self.state.repo.clone();
5262 let coordination = self.coordinate(&repo).await?;
5263 if winner.worktree.exists() {
5264 let dirty = git::git(
5267 &winner.worktree,
5268 &["status", "--porcelain", "--untracked-files=all"],
5269 )
5270 .await?;
5271 let only_withheld = dirty.lines().all(|l| {
5272 l.strip_prefix("?? ")
5273 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5274 });
5275 if !only_withheld {
5276 bail!(
5277 "`{}` has uncommitted changes; refusing to touch it — commit or \
5278 discard them first",
5279 winner.worktree.display()
5280 );
5281 }
5282 git::worktree_remove(&self.state.repo, &winner.worktree)
5283 .await
5284 .ok();
5285 }
5286 let fix_worktree = self.state.worktree_root().join("operator-fix");
5287 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5288 git::git(
5289 &self.state.repo,
5290 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5291 )
5292 .await
5293 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5294 if !git::is_clean(&fix_worktree).await? {
5295 git::worktree_remove(&self.state.repo, &fix_worktree)
5296 .await
5297 .ok();
5298 bail!(
5299 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5300 winner.branch
5301 );
5302 }
5303 let repo_for_setup = self.state.repo.clone();
5304 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5305 Ok(()) => {
5306 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5307 }
5308 Err(e) => Err(e),
5309 };
5310 if let Err(e) = setup {
5311 git::worktree_remove(&self.state.repo, &fix_worktree)
5312 .await
5313 .ok();
5314 return Err(e);
5315 }
5316
5317 let run_id = self.state.id.clone();
5318 let prompts = self.state.config.prompts.clone();
5319 let language = self.state.config.graph.language.clone();
5320 let sessions = self.state.config.graph.sessions;
5321 let artifacts = agent::artifacts_dir(&self.state.dir());
5322 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5323 .findings
5324 .iter()
5325 .map(|f| Finding {
5326 id: f.id.clone(),
5327 severity: f.severity,
5328 file: f.file.clone(),
5329 line: f.line,
5330 title: f.title.clone(),
5331 detail: f.detail.clone(),
5332 })
5333 .collect();
5334 let fix_prompt = prompt::operator_fix_in(
5335 &self.state.instruction,
5336 &finding_list,
5337 reason,
5338 &stale_details,
5339 &head_at_request,
5340 &language,
5341 &self.state.config.graph.github_language,
5342 );
5343 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5344 drop(coordination);
5345 let (job, seat, out) = self
5346 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5347 prompt: fix_prompt.clone(),
5348 spec,
5349 seat,
5350 cwd: fix_worktree.clone(),
5351 timeout,
5352 allow_write: true,
5353 sessions,
5354 artifacts: artifacts.clone(),
5355 stem: "operator-fix".to_owned(),
5356 handover: None,
5357 })
5358 .await;
5359 let agent_id = seat.agent.clone();
5360
5361 let mut fix = FixRecord {
5362 agent: agent_id,
5363 addressed: Vec::new(),
5364 rejected: Vec::new(),
5365 notes: String::new(),
5366 committed: false,
5367 failed: None,
5368 duration_ms: 0,
5369 continuation: None,
5370 };
5371 let mut final_seat = seat.clone();
5372 match out {
5373 AgentOutcome::Ok(o) => {
5374 fix.duration_ms = o.duration_ms;
5375 let parsed = verdict::extract_json::<FixReport>(&o.text);
5376 let incomplete_reason = match &parsed {
5377 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5378 "the reply parsed, but it reported a command whose own CLI \
5379 never confirmed an exit status"
5380 .to_owned(),
5381 ),
5382 Ok(_) => None,
5383 Err(e) => Some(e.to_string()),
5384 };
5385 match incomplete_reason {
5386 None => {
5387 let report = parsed.expect("checked Ok above");
5388 fix.addressed = report.addressed;
5389 fix.rejected = report.rejected;
5390 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5391 }
5392 Some(reason) => {
5393 let (resumed_seat, resolved, failure, cont) = self
5394 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5395 .await;
5396 fix.duration_ms += cont.cumulative_wait_ms;
5397 fix.continuation = Some(cont);
5398 final_seat = resumed_seat;
5399 match resolved {
5400 Some(report) => {
5401 fix.addressed = report.addressed;
5402 fix.rejected = report.rejected;
5403 fix.notes =
5404 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5405 }
5406 None => fix.failed = failure,
5407 }
5408 }
5409 }
5410 }
5411 AgentOutcome::Dropped(o) => {
5412 fix.duration_ms = o.duration_ms;
5413 let why = o
5414 .dropped
5415 .as_ref()
5416 .map(|d| d.why.as_str())
5417 .unwrap_or("the CLI ended the stream without delivering its answer");
5418 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5419 }
5420 AgentOutcome::Quota(o) => {
5421 self.state.quota.push(QuotaLoss {
5422 seat: final_seat.key.clone(),
5423 node: "fix".to_owned(),
5424 at: Timestamp::now(),
5425 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5426 });
5427 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5428 }
5429 AgentOutcome::Failed(e) => fix.failed = Some(e),
5430 AgentOutcome::ResourceBlocked(e) => {
5431 fix.failed = Some(format!("the fixer was not started: {e}"));
5432 }
5433 }
5434 if fix.continuation.is_none() {
5435 fix.continuation = Some(ContinuationRecord::not_needed());
5436 }
5437 self.state.seats.insert(final_seat.key.clone(), final_seat);
5438
5439 let rescue_message = format!(
5440 "magi: operator-selected fix ({}) (uncommitted work)",
5441 self.state.operator_fixes[request_index]
5442 .findings
5443 .iter()
5444 .map(|f| f.id.as_str())
5445 .collect::<Vec<_>>()
5446 .join(", ")
5447 );
5448 let repo = self.state.repo.clone();
5449 let coordination = self.coordinate(&repo).await?;
5450 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5451 self.state.note_withheld("fix", &r.withheld);
5452 }
5453 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5454 fix.committed = after != head_at_request;
5455 git::worktree_remove(&self.state.repo, &fix_worktree)
5456 .await
5457 .ok();
5458 drop(coordination);
5459
5460 self.state.event(
5461 "fix",
5462 match &fix.failed {
5463 Some(reason) => format!(
5464 "operator fix: adoption report was lost ({reason}); {}",
5465 if fix.committed {
5466 "committed"
5467 } else {
5468 "NO new commit"
5469 }
5470 ),
5471 None => format!(
5472 "operator fix: {} addressed, {} rejected, {}",
5473 fix.addressed.len(),
5474 fix.rejected.len(),
5475 if fix.committed {
5476 "committed"
5477 } else {
5478 "NO new commit"
5479 }
5480 ),
5481 },
5482 );
5483
5484 for f in &mut self.state.operator_fixes[request_index].findings {
5491 f.outcome = if fix.failed.is_some() {
5492 OperatorFixOutcome::Unreported
5493 } else if fix.addressed.contains(&f.id) {
5494 OperatorFixOutcome::Addressed
5495 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5496 OperatorFixOutcome::Rejected { why: r.why.clone() }
5497 } else {
5498 OperatorFixOutcome::Unreported
5499 };
5500 }
5501
5502 let committed = fix.committed;
5503 if committed {
5504 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5505 }
5506 self.state.operator_fixes[request_index].fix = Some(fix);
5507 self.state.save()?;
5510
5511 if committed {
5512 self.state.event(
5513 "fix",
5514 format!(
5515 "operator fix committed {}; opening a follow-up review-only run",
5516 short(&after)
5517 ),
5518 );
5519 let origin =
5522 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5523 match Self::review(
5524 &self.state.repo,
5525 &winner.branch,
5526 self.state.config.clone(),
5527 origin,
5528 )
5529 .await
5530 {
5531 Ok(mut follow_up) => {
5532 follow_up.state.event(
5533 "start",
5534 format!(
5535 "requested by an operator fix on run {} for finding(s) {}",
5536 self.state.id,
5537 self.state.operator_fixes[request_index]
5538 .findings
5539 .iter()
5540 .map(|f| f.id.as_str())
5541 .collect::<Vec<_>>()
5542 .join(", "),
5543 ),
5544 );
5545 follow_up.state.save()?;
5546 let follow_up_id = follow_up.state.id.clone();
5547 let adopted = match crate::direct::adopt(
5550 &follow_up.state,
5551 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5552 ) {
5553 Ok(a) => a,
5554 Err(e) => {
5555 self.state.event(
5558 "fix",
5559 format!(
5560 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5561 ),
5562 );
5563 self.state.save()?;
5564 return Ok(());
5565 }
5566 };
5567 let executed = follow_up.execute().await;
5568 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5569 if let Some(a) = adopted {
5570 a.finish(&follow_up.state, executed);
5571 }
5572 if let Some(e) = failure {
5573 self.state.event(
5574 "fix",
5575 format!(
5576 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5577 ),
5578 );
5579 }
5580 self.state.operator_fixes[request_index].follow_up_review_run =
5581 Some(follow_up_id);
5582 }
5583 Err(e) => {
5584 self.state.event(
5585 "fix",
5586 format!("committed the fix but could not open a follow-up review: {e:#}"),
5587 );
5588 }
5589 }
5590 self.state.save()?;
5591 }
5592
5593 Ok(())
5594 }
5595
5596 async fn ask_fixer(
5616 &mut self,
5617 winner: &Candidate,
5618 node: &'static str,
5619 round: Option<usize>,
5620 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5621 ) -> (SeatJob, SeatState, AgentOutcome) {
5622 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5623 self.ask_fixer_among(attempts, node, round, None, build)
5624 .await
5625 }
5626
5627 async fn ask_fixer_among(
5632 &mut self,
5633 attempts: Vec<(AgentSpec, String)>,
5634 node: &'static str,
5635 round: Option<usize>,
5636 restore: Option<(&Path, &str)>,
5637 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5638 ) -> (SeatJob, SeatState, AgentOutcome) {
5639 let run_id = self.state.id.clone();
5640 let prompts = self.state.config.prompts.clone();
5641 let cache = self.state.config.cache_dir();
5642 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5643 let mut last = None;
5644 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5645 let seat = self.seat(&key, &spec.id);
5646 let mut job = build(spec.clone(), seat);
5647 if i > 0 {
5648 job.stem = format!("{}-{}", job.stem, spec.id);
5649 }
5650 let ctx = WaveCtx {
5651 carry_seats: false,
5652 pauses: Some((&self.pause, &self.interrupt)),
5653 run: &run_id,
5654 node,
5655 prompts: &prompts,
5656 cache: cache.as_deref(),
5657 round,
5658 };
5659 let (seat, out) =
5660 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5661 let advances = match &out {
5664 AgentOutcome::Ok(o) => agent::output_advances(o),
5665 AgentOutcome::ResourceBlocked(_) => false,
5666 _ => true,
5667 };
5668 if let Some(next) = ids.get(i + 1)
5669 && advances
5670 {
5671 self.state.seats.insert(seat.key.clone(), seat.clone());
5672 let class =
5673 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5674 let reason = match &out {
5675 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5676 other => fail_reason(other),
5677 };
5678 if let Some((wt, rev)) = restore
5682 && let Err(e) = self.graft_restore(wt, rev).await
5683 {
5684 let why =
5685 format!("could not restore the worktree before the next fixer: {e:#}");
5686 tracing::warn!("{why}");
5687 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5688 AgentOutcome::ResourceBlocked(why)
5689 } else {
5690 AgentOutcome::Failed(why)
5691 };
5692 last = Some((job, seat, out));
5693 break;
5694 }
5695 record_handover(
5696 &mut self.state,
5697 node,
5698 &seat.key,
5699 &spec.id,
5700 next,
5701 &class,
5702 &reason,
5703 );
5704 continue;
5705 }
5706 last = Some((job, seat, out));
5707 break;
5708 }
5709 last.expect("the fixer chain always has an entry")
5710 }
5711
5712 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5723 use crate::recovery::{Mode, Repair};
5724 if !self.state.reviews.is_empty() {
5725 return Ok(());
5726 }
5727 let Some(mut rec) = self.state.recovery.clone() else {
5728 return Ok(());
5729 };
5730 if rec.mode != Mode::VerifyRepair {
5731 return Ok(());
5732 }
5733 let wt = winner.worktree.clone();
5734 let mut head = git::rev_parse(&wt, "HEAD").await?;
5735 match rec.repair.clone() {
5736 Repair::Done { .. } => return Ok(()),
5737 Repair::WaitingForRestore { before } => {
5738 self.graft_restore(&wt, &before).await?;
5739 head = before;
5740 rec.repair = Repair::NotStarted;
5741 self.state.recovery = Some(rec.clone());
5742 self.state.save()?;
5743 }
5744 Repair::Started { before } => {
5745 if let Err(e) = self.graft_restore(&wt, &before).await {
5749 if e.is::<crate::repo_lock::ResourceBlocked>() {
5750 return Err(e);
5751 }
5752 return self
5753 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5754 .await;
5755 }
5756 rec.repair = Repair::Done {
5757 before: before.clone(),
5758 after: before,
5759 committed: false,
5760 failed: Some("interrupted; rolled back".to_owned()),
5761 };
5762 self.state
5763 .event("recovery", "the repair was interrupted and rolled back");
5764 self.state.recovery = Some(rec);
5765 self.state.save()?;
5766 return Ok(());
5767 }
5768 Repair::NotStarted => {}
5769 }
5770 if crate::recovery::recheck(&mut rec, &head) {
5771 self.state.event("recovery", rec.summary());
5772 self.state.recovery = Some(rec);
5773 self.state.save()?;
5774 return Ok(());
5775 }
5776 rec.repair = Repair::Started {
5777 before: head.clone(),
5778 };
5779 self.state.recovery = Some(rec.clone());
5780 self.state.save()?;
5781
5782 let language = self.state.config.graph.language.clone();
5783 let sessions = self.state.config.graph.sessions;
5784 let artifacts = agent::artifacts_dir(&self.state.dir());
5785 let fix_prompt = prompt::recovery_fix(
5786 &self.state.instruction,
5787 &rec,
5788 self.state.config.graph.review_rounds,
5789 &language,
5790 );
5791 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5792 self.state.event(
5793 "recovery",
5794 format!(
5795 "repair: the fixer starts from the failed verification on {}",
5796 short(&head)
5797 ),
5798 );
5799 let (_job, seat, out) = self
5800 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5801 prompt: fix_prompt.clone(),
5802 spec,
5803 seat,
5804 cwd: wt.clone(),
5805 timeout,
5806 allow_write: true,
5807 sessions,
5808 artifacts: artifacts.clone(),
5809 stem: "fix-recovery".to_owned(),
5810 handover: None,
5811 })
5812 .await;
5813 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5814 rec.repair = Repair::WaitingForRestore { before: head };
5815 self.state.recovery = Some(rec);
5816 return self.stop_for_resource();
5817 }
5818 let failed = match &out {
5819 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5820 AgentOutcome::Quota(o) => {
5821 self.state.quota.push(QuotaLoss {
5822 seat: seat.key.clone(),
5823 node: "fix".to_owned(),
5824 at: Timestamp::now(),
5825 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5826 });
5827 Some("rate limited (quota); the fixer could not run".to_owned())
5828 }
5829 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5830 AgentOutcome::Failed(e) => Some(e.clone()),
5831 };
5832 self.state.seats.insert(seat.key.clone(), seat);
5833 let coordination = self.coordinate(&wt).await?;
5834 let failed = match failed {
5835 Some(why) => {
5837 if let Err(e) = git::restore_to(&wt, &head).await {
5838 return self
5839 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5840 .await;
5841 }
5842 Some(why)
5843 }
5844 None => {
5845 if let Ok(r) = git::rescue_commit(
5846 &wt,
5847 "magi: recovery repair of the failed verification (uncommitted work)",
5848 )
5849 .await
5850 {
5851 self.state.note_withheld("fix", &r.withheld);
5852 }
5853 None
5854 }
5855 };
5856 let after = git::rev_parse(&wt, "HEAD").await?;
5857 drop(coordination);
5858 self.state.event(
5859 "recovery",
5860 match &failed {
5861 Some(why) => format!("repair: nothing kept ({why})"),
5862 None if after != head => {
5863 format!("repair: committed {}", short(&after))
5864 }
5865 None => "repair: the fixer made no commit".to_owned(),
5866 },
5867 );
5868 rec.repair = Repair::Done {
5869 before: head.clone(),
5870 committed: after != head,
5871 after,
5872 failed,
5873 };
5874 self.state.recovery = Some(rec);
5875 self.state.save()?;
5876 Ok(())
5877 }
5878
5879 async fn stop_recovery(
5882 &mut self,
5883 mut rec: crate::recovery::RecoveryRecord,
5884 head: &str,
5885 why: String,
5886 ) -> Result<()> {
5887 rec.repair = crate::recovery::Repair::Started {
5889 before: head.to_owned(),
5890 };
5891 self.state.recovery = Some(rec);
5892 self.state
5893 .event("recovery", format!("repair could not be undone: {why}"));
5894 self.state.status = RunStatus::Blocked;
5895 self.state.save()?;
5896 bail!("recovery repair could not be rolled back: {why}")
5897 }
5898
5899 async fn review_loop(&mut self) -> Result<()> {
5900 loop {
5901 if self
5902 .state
5903 .base_sync
5904 .as_ref()
5905 .is_some_and(|s| s.conflict.is_some())
5906 {
5907 return Ok(());
5908 }
5909 if self.catchup_repairable() {
5910 let record = self.state.reviews.last().expect("repairable round").clone();
5911 let winner = self
5912 .state
5913 .winner()
5914 .cloned()
5915 .context("catch-up repair has no winner")?;
5916 if !self.fix_review_round(&winner, record).await? {
5917 return Ok(());
5918 }
5919 }
5920 self.review_loop_pass().await?;
5921 if !self.catchup_repairable() {
5922 return Ok(());
5923 }
5924 }
5925 }
5926
5927 fn catchup_repairable(&self) -> bool {
5930 self.state.reviews.len() < self.state.config.graph.review_rounds
5931 && self.state.reviews.last().is_some_and(|r| {
5932 !r.clean
5933 && r.blocking == 0
5934 && r.fix.is_none()
5935 && r.e2e_status() == E2eStatus::Failed
5936 })
5937 }
5938
5939 async fn review_loop_pass(&mut self) -> Result<()> {
5940 if self
5945 .state
5946 .base_sync
5947 .as_ref()
5948 .is_some_and(|s| s.conflict.is_some())
5949 {
5950 return Ok(());
5951 }
5952 let run_id = self.state.id.clone();
5957 let prompts = self.state.config.prompts.clone();
5958 let Some(winner) = self.state.winner().cloned() else {
5959 return Ok(());
5960 };
5961 let max_rounds = self.state.config.graph.review_rounds;
5962 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5972 if status == RunStatus::Gating {
5975 self.record_contested_handoff();
5976 }
5977 self.state.status = status;
5978 self.state.save()?;
5979 return Ok(());
5980 }
5981 self.state.status = RunStatus::Reviewing;
5982 if self.state.reviews.last().is_some_and(|r| {
5992 r.e2e_status() == E2eStatus::ResourceBlocked
5993 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5994 }) {
5995 let shell = self.state.config.shell();
5996 return self
5997 .stop_unresolved_review(
5998 "the last round's own verification never resolved",
5999 &shell,
6000 &winner.worktree,
6001 )
6002 .await;
6003 }
6004
6005 self.recovery_repair(&winner).await?;
6006 if self.resource_halt()? {
6007 return Ok(());
6008 }
6009
6010 let repo = self.state.repo.clone();
6011 let root = self.state.worktree_root();
6012 let language = self.state.config.graph.language.clone();
6013 let sessions = self.state.config.graph.sessions;
6014 let artifacts = self.merge_repair_artifacts();
6015 let base = self.landing_base();
6016 let base_short = short(&base);
6017 let reviewers = self.roles.reviewers.clone();
6018 let shell = self.state.config.shell();
6019
6020 for round in (self.state.reviews.len() + 1)..=max_rounds {
6021 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6022 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6023 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6024 let prev_verification = self
6033 .state
6034 .reviews
6035 .last()
6036 .and_then(|r| r.verification_summary(&head));
6037 let prior_section = match (&self.state.recovery, round) {
6040 (Some(rec), 1) if self.state.reviews.is_empty() => {
6041 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6042 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6043 Some(crate::recovery::head_of(
6044 &git::diff(&winner.worktree, reviewed, "HEAD")
6045 .await
6046 .unwrap_or_default(),
6047 crate::recovery::SINCE_DIFF_MAX,
6048 ))
6049 }
6050 _ => None,
6051 };
6052 Some(prompt::recovery_review(rec, since.as_deref()))
6053 }
6054 _ => None,
6055 };
6056
6057 let mut jobs = Vec::new();
6061 let coordination = self.coordinate(&repo).await?;
6062 for (r, spec) in reviewers.iter().cloned().enumerate() {
6063 let wt = root.join(format!("review-{}", r + 1));
6064 if wt.exists() {
6067 git::reset_detached(&wt, &head).await?;
6068 } else {
6069 git::worktree_add_detached(&repo, &wt, &head).await?;
6070 }
6071 self.setup_seat_worktree(&repo, &wt, false).await?;
6072 let seat_key = format!("review-{}", r + 1);
6073 let spec = pick_start_spec(
6076 &self.roles.reviewer_roster,
6077 spec,
6078 self.state.seat_history.get(&seat_key),
6079 );
6080 let seat = self.seat(&seat_key, &spec.id);
6081 jobs.push(SeatJob {
6082 prompt: prompt::review(&prompt::ReviewCtx {
6083 instruction: &self.state.instruction,
6084 branch: &winner.branch,
6085 base_short: &base_short,
6086 stat: &stat,
6087 patch: &patch,
6088 verification: prev_verification.as_ref(),
6089 prior: prior_section.as_deref(),
6090 reviewers: reviewers.len(),
6091 round,
6092 rounds: max_rounds,
6093 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6096 lens: Lens::for_seat(r),
6097 language: &language,
6098 github_language: &self.state.config.graph.github_language,
6099 }),
6100 spec,
6101 seat,
6102 cwd: wt,
6103 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6104 allow_write: false,
6105 sessions,
6106 artifacts: artifacts.clone(),
6107 stem: format!("review-{round}-{}", r + 1),
6108 handover: None,
6109 });
6110 }
6111
6112 drop(coordination);
6113 self.state.event(
6114 "review",
6115 format!(
6116 "round {round}: {} reviewers on {}",
6117 jobs.len(),
6118 short(&head)
6119 ),
6120 );
6121 let mut quota_losses = Vec::new();
6122 let review_retries = self.state.config.graph.retries;
6123 let review_cache = self.state.config.cache_dir();
6124 let ctx = WaveCtx {
6125 carry_seats: true,
6126 pauses: Some((&self.pause, &self.interrupt)),
6127 run: &run_id,
6128 node: "review",
6129 prompts: &prompts,
6130 cache: review_cache.as_deref(),
6131 round: Some(round),
6132 };
6133 let results = ask_json_wave::<Review>(
6134 jobs,
6135 Arc::clone(&self.sem),
6136 review_retries,
6137 &self.roles.reviewer_roster,
6138 &ctx,
6139 &mut quota_losses,
6140 &mut self.state,
6141 &|_: &Review| Ok(()),
6142 )
6143 .await;
6144 if self.state.resource_block.is_some() {
6145 return self.stop_for_resource();
6146 }
6147 let round_quota_missing = quota_losses.len();
6151 self.state.quota.extend(quota_losses);
6152
6153 let mut records = Vec::new();
6154 let mut all_findings = Vec::new();
6155 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6156 let agent_id = seat.agent.clone();
6157 self.state.seats.insert(seat.key.clone(), seat);
6158 let mut record = ReviewRecord {
6159 reviewer: r + 1,
6160 agent: agent_id,
6161 summary: String::new(),
6162 findings: Vec::new(),
6163 vote: None,
6164 failed: None,
6165 duration_ms: 0,
6166 attempts,
6172 };
6173 match res {
6174 Ok((review, out)) => {
6175 record.summary =
6183 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6184 record.vote = Some(review.vote);
6185 record.duration_ms = out.duration_ms;
6186 for (n, mut f) in review.findings.into_iter().enumerate() {
6187 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6190 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6191 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6192 f.file = f
6198 .file
6199 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6200 all_findings.push(f.clone());
6201 record.findings.push(f);
6202 }
6203 self.state.event(
6204 "review",
6205 format!(
6206 "round {round}: reviewer {} voted {} with {} finding(s)",
6207 r + 1,
6208 review.vote.label(),
6209 record.findings.len()
6210 ),
6211 );
6212 }
6213 Err(e) => {
6214 record.failed = Some(e.to_string());
6215 self.state.event(
6216 "review",
6217 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6218 );
6219 }
6220 }
6221 records.push(record);
6222 }
6223
6224 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6231 let vote_split =
6232 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6233 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6234 if vote_split {
6235 self.state.event(
6236 "review",
6237 format!(
6238 "round {round}: votes split ({}) — one round of reconsideration",
6239 initial_votes
6240 .iter()
6241 .map(|v| v.label())
6242 .collect::<Vec<_>>()
6243 .join(", ")
6244 ),
6245 );
6246 let panel: Vec<ReviewSeatReport<'_>> = records
6249 .iter()
6250 .filter_map(|r| {
6251 r.vote.map(|vote| ReviewSeatReport {
6252 reviewer: r.reviewer,
6253 vote,
6254 summary: &r.summary,
6255 findings: &r.findings,
6256 })
6257 })
6258 .collect();
6259
6260 let mut jobs = Vec::new();
6261 let mut seats_at = Vec::new();
6262 for (r, spec) in reviewers.iter().cloned().enumerate() {
6263 if records[r].vote.is_none() {
6267 continue;
6268 }
6269 let wt = root.join(format!("review-{}", r + 1));
6270 let seat_key = format!("review-{}", r + 1);
6271 let spec = self.occupant(&seat_key, spec);
6272 let seat = self.seat(&seat_key, &spec.id);
6273 let build = |with_patch: bool| {
6281 prompt::review_reconsider(&ReviewReconsiderCtx {
6282 instruction: &self.state.instruction,
6283 reviewer: r + 1,
6284 lens: Lens::for_seat(r),
6285 panel: &panel,
6286 patch: with_patch.then_some(ReviewPatch {
6287 branch: &winner.branch,
6288 base_short: &base_short,
6289 stat: &stat,
6290 patch: &patch,
6291 }),
6292 round,
6293 rounds: max_rounds,
6294 language: &language,
6295 })
6296 };
6297 let full = build(true);
6298 let prompt = if has_context(&spec, &seat, sessions) {
6299 build(false)
6300 } else {
6301 full.clone()
6302 };
6303 jobs.push(SeatJob {
6304 prompt,
6305 spec,
6306 seat,
6307 cwd: wt,
6308 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6309 allow_write: false,
6310 sessions,
6311 artifacts: artifacts.clone(),
6312 stem: format!("review-{round}-reconsider-{}", r + 1),
6313 handover: Some(full),
6314 });
6315 seats_at.push(r);
6316 }
6317
6318 let mut recon_quota_losses = Vec::new();
6319 let recon_cache = self.state.config.cache_dir();
6320 let recon_ctx = WaveCtx {
6321 carry_seats: true,
6322 pauses: Some((&self.pause, &self.interrupt)),
6323 run: &run_id,
6324 node: "review",
6325 prompts: &prompts,
6326 cache: recon_cache.as_deref(),
6327 round: Some(round),
6328 };
6329 let recon_results = ask_json_wave::<ReviewRevote>(
6330 jobs,
6331 Arc::clone(&self.sem),
6332 review_retries,
6333 &self.roles.reviewer_roster,
6334 &recon_ctx,
6335 &mut recon_quota_losses,
6336 &mut self.state,
6337 &|_: &ReviewRevote| Ok(()),
6338 )
6339 .await;
6340 if self.state.resource_block.is_some() {
6341 return self.stop_for_resource();
6342 }
6343 self.state.quota.extend(recon_quota_losses);
6344
6345 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6346 let agent_id = seat.agent.clone();
6347 self.state.seats.insert(seat.key.clone(), seat);
6348 let mut rec = ReviewRevoteRecord {
6349 reviewer: r + 1,
6350 agent: agent_id,
6351 vote: None,
6352 reason: String::new(),
6353 failed: None,
6354 };
6355 match res {
6356 Ok((rv, _)) => {
6357 rec.vote = Some(rv.vote);
6358 rec.reason =
6359 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6360 self.state.event(
6361 "review",
6362 format!(
6363 "round {round}: reviewer {} revoted {}",
6364 r + 1,
6365 rv.vote.label()
6366 ),
6367 );
6368 }
6369 Err(e) => {
6370 rec.failed = Some(e.to_string());
6371 self.state.event(
6372 "review",
6373 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6374 );
6375 }
6376 }
6377 reconsideration.push(rec);
6378 }
6379 } else if initial_votes.len() > 1 {
6380 self.state.event(
6381 "review",
6382 format!(
6383 "round {round}: votes agreed ({}) — no reconsideration",
6384 initial_votes[0].label()
6385 ),
6386 );
6387 }
6388
6389 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6390 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6391 let defer_e2e =
6402 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6403 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6404 let reason =
6405 format!("{blocking} blocking finding(s) already required a fix this round");
6406 self.state.event(
6407 "verify",
6408 format!(
6409 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6410 {}); it will run once a round has none left",
6411 short(&head)
6412 ),
6413 );
6414 (Vec::new(), false, true, Some(reason))
6415 } else {
6416 let e2e_commands = self.state.config.verify.e2e.clone();
6417 let cache_dir = self.state.config.cache_dir();
6418 let context = format!("round {round}");
6419 let wait = self.repository_wait();
6420 let (e2e, verify_retried) = with_cache_lease(
6421 &mut self.state,
6422 cache_dir.as_deref(),
6423 "e2e",
6424 "e2e",
6425 &winner.worktree,
6426 &head,
6427 verify_timeout,
6428 &context,
6429 &wait,
6430 |state, budget| {
6431 let shell = shell.clone();
6432 let e2e_commands = e2e_commands.clone();
6433 let worktree = winner.worktree.clone();
6434 let context = context.clone();
6435 async move {
6436 run_e2e_with_retry(
6437 state,
6438 &shell,
6439 &e2e_commands,
6440 &worktree,
6441 budget,
6442 &context,
6443 )
6444 .await
6445 }
6446 },
6447 )
6448 .await?;
6449 (e2e, verify_retried, false, None)
6450 };
6451
6452 let expected = records.len();
6453 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6454 let incomplete = answered < expected;
6455 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6456 let policy = self.state.config.graph.incomplete_review;
6457 let clean = round_is_clean(
6458 blocking,
6459 e2e_ok,
6460 answered,
6461 expected,
6462 round_quota_missing,
6463 policy,
6464 );
6465
6466 let mut round_record = ReviewRound {
6467 round,
6468 head: head.clone(),
6469 verified_head: None,
6470 verified_at: None,
6471 reviews: records,
6472 e2e,
6473 verify_retried,
6474 e2e_deferred,
6475 e2e_defer_reason,
6476 fix: None,
6477 blocking,
6478 answered,
6479 expected,
6480 clean,
6481 progressed: false,
6482 vote_split,
6483 reconsideration,
6484 verdict: None,
6485 };
6486 round_record.verdict = ReviewVote::worst(
6490 round_record
6491 .final_votes()
6492 .into_iter()
6493 .map(|(_, _, vote)| vote),
6494 );
6495 if !matches!(
6506 round_record.e2e_status(),
6507 E2eStatus::Deferred | E2eStatus::NotConfigured
6508 ) {
6509 round_record.verified_head = Some(head.clone());
6510 round_record.verified_at = Some(Timestamp::now());
6511 }
6512
6513 if incomplete {
6514 let missing: Vec<String> = round_record
6515 .reviews
6516 .iter()
6517 .filter(|r| r.failed.is_some())
6518 .map(|r| format!("review-{}", r.reviewer))
6519 .collect();
6520 self.state.event(
6521 "review",
6522 format!(
6523 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6524 missing.join(", ")
6525 ),
6526 );
6527 }
6528
6529 if clean {
6530 self.state.event(
6531 "review",
6532 if incomplete && policy == IncompleteReviewPolicy::Warn {
6533 format!(
6534 "round {round}: clean (warn policy, incomplete panel) — no \
6535 blocking findings from the seats that answered, verification green"
6536 )
6537 } else if incomplete {
6538 format!(
6539 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6540 quorum) — no blocking findings from the seats that answered, \
6541 verification green",
6542 expected - answered
6543 )
6544 } else {
6545 format!("round {round}: clean — no blocking findings, verification green")
6546 },
6547 );
6548 self.state.reviews.push(round_record);
6549 self.state.status = RunStatus::Gating;
6550 self.state.save()?;
6551 return Ok(());
6552 }
6553
6554 if incomplete && blocking == 0 && e2e_ok {
6562 self.state.reviews.push(round_record);
6563 self.state.save()?;
6564 if round == max_rounds {
6565 self.state.status = RunStatus::Blocked;
6566 self.state.event(
6567 "review",
6568 format!(
6569 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6570 refusing to call it clean",
6571 expected - answered
6572 ),
6573 );
6574 return Ok(());
6575 }
6576 continue;
6577 }
6578
6579 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6591 self.state.reviews.push(round_record);
6592 return self
6593 .stop_unresolved_review(
6594 "the round's own verification could not run",
6595 &shell,
6596 &winner.worktree,
6597 )
6598 .await;
6599 }
6600
6601 if round == max_rounds {
6602 self.state.reviews.push(round_record);
6603 return self
6604 .stop_reviewing(
6605 &format!(
6606 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6607 ),
6608 &shell,
6609 &winner.worktree,
6610 )
6611 .await;
6612 }
6613
6614 if !self.fix_review_round(&winner, round_record).await? {
6615 return Ok(());
6616 }
6617 }
6618 Ok(())
6619 }
6620
6621 async fn fix_review_round(
6624 &mut self,
6625 winner: &Candidate,
6626 mut round_record: ReviewRound,
6627 ) -> Result<bool> {
6628 let round = round_record.round;
6629 let shell = self.state.config.shell();
6630 let step = self.run_fix_round(winner, &mut round_record).await?;
6631 if matches!(step, FixStep::Resource) {
6632 self.stop_for_resource()?;
6633 return Ok(false);
6634 }
6635 if self.state.reviews.len() < round {
6636 self.state.reviews.push(round_record);
6637 } else {
6638 self.state.reviews[round - 1] = round_record;
6639 }
6640 self.state.save()?;
6641
6642 if matches!(step, FixStep::Recorded { report_lost: true }) {
6655 self.stop_reviewing(
6656 "the fixer's adoption report never came back, even after resuming its \
6657 own seat; refusing to start another round against the same worktree \
6658 while that is unresolved",
6659 &shell,
6660 &winner.worktree,
6661 )
6662 .await?;
6663 return Ok(false);
6664 }
6665
6666 let streak = self
6667 .state
6668 .reviews
6669 .iter()
6670 .rev()
6671 .take_while(|r| !r.progressed)
6672 .count();
6673 if streak >= STAGNANT_LIMIT {
6674 self.stop_reviewing(
6675 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6676 &shell,
6677 &winner.worktree,
6678 )
6679 .await?;
6680 return Ok(false);
6681 }
6682 Ok(true)
6683 }
6684
6685 async fn run_fix_round(
6687 &mut self,
6688 winner: &Candidate,
6689 round_record: &mut ReviewRound,
6690 ) -> Result<FixStep> {
6691 let round = round_record.round;
6692 let max_rounds = self.state.config.graph.review_rounds;
6693 let run_id = self.state.id.clone();
6694 let prompts = self.state.config.prompts.clone();
6695 let language = self.state.config.graph.language.clone();
6696 let sessions = self.state.config.graph.sessions;
6697 let artifacts = self.merge_repair_artifacts();
6698 let base = self.landing_base();
6699 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6700 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6701 let this_round_verification = round_record.verification_summary(&head);
6702 let all_findings: Vec<_> = round_record
6703 .reviews
6704 .iter()
6705 .flat_map(|r| r.findings.iter().cloned())
6706 .collect();
6707 let blocking_findings: Vec<_> = all_findings
6710 .iter()
6711 .filter(|f| f.severity.blocks())
6712 .cloned()
6713 .collect();
6714 let fix_prompt = prompt::fix_in(
6715 &self.state.instruction,
6716 &blocking_findings,
6717 this_round_verification.as_ref(),
6718 round,
6719 max_rounds,
6720 &language,
6721 &self.state.config.graph.github_language,
6722 );
6723 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6724 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6725 let (job, seat, out) = self
6726 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6727 prompt: fix_prompt.clone(),
6728 spec,
6729 seat,
6730 cwd: winner.worktree.clone(),
6731 timeout,
6732 allow_write: true,
6733 sessions,
6734 artifacts: artifacts.clone(),
6735 stem: format!("fix-{round}"),
6736 handover: None,
6737 })
6738 .await;
6739 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6742 return Ok(FixStep::Resource);
6743 }
6744 let agent_id = seat.agent.clone();
6745
6746 let mut fix = FixRecord {
6747 agent: agent_id,
6748 addressed: Vec::new(),
6749 rejected: Vec::new(),
6750 notes: String::new(),
6751 committed: false,
6752 failed: None,
6753 duration_ms: 0,
6754 continuation: None,
6755 };
6756 let mut continuation = ContinuationRecord::not_needed();
6757 let mut final_seat = seat.clone();
6758 match out {
6759 AgentOutcome::Ok(o) => {
6760 fix.duration_ms = o.duration_ms;
6761 let parsed = verdict::extract_json::<FixReport>(&o.text);
6762 let incomplete_reason = match &parsed {
6769 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6770 "the reply parsed, but it reported a command whose own CLI never \
6771 confirmed an exit status"
6772 .to_owned(),
6773 ),
6774 Ok(_) => None,
6775 Err(e) => Some(e.to_string()),
6776 };
6777 match incomplete_reason {
6778 None => {
6779 let report = parsed.expect("checked Ok above");
6780 fix.addressed = report.addressed;
6781 fix.rejected = report.rejected;
6782 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6783 }
6784 Some(reason) => {
6785 let (resumed_seat, resolved, failure, cont) = self
6786 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6787 .await;
6788 fix.duration_ms += cont.cumulative_wait_ms;
6789 continuation = cont;
6790 final_seat = resumed_seat;
6791 match resolved {
6792 Some(report) => {
6793 fix.addressed = report.addressed;
6794 fix.rejected = report.rejected;
6795 fix.notes =
6796 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6797 }
6798 None => fix.failed = failure,
6799 }
6800 }
6801 }
6802 }
6803 AgentOutcome::Dropped(o) => {
6805 fix.duration_ms = o.duration_ms;
6806 let why = o
6807 .dropped
6808 .as_ref()
6809 .map(|d| d.why.as_str())
6810 .unwrap_or("the CLI ended the stream without delivering its answer");
6811 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6812 }
6813 AgentOutcome::Quota(o) => {
6814 self.state.quota.push(QuotaLoss {
6815 seat: final_seat.key.clone(),
6816 node: "fix".to_owned(),
6817 at: Timestamp::now(),
6818 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6819 });
6820 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6821 }
6822 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6823 }
6824 fix.continuation = Some(continuation);
6825 self.state.seats.insert(final_seat.key.clone(), final_seat);
6826 let coordination = self.coordinate(&winner.worktree).await?;
6827 if let Ok(r) = git::rescue_commit(
6828 &winner.worktree,
6829 &format!("magi: review round {round} fixes (uncommitted work)"),
6830 )
6831 .await
6832 {
6833 self.state.note_withheld("fix", &r.withheld);
6834 }
6835 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6836 fix.committed = after != before;
6837 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6845 drop(coordination);
6846 let progressed = diff_after != patch;
6847 let commit_note = if fix.committed {
6848 "committed"
6849 } else {
6850 "NO new commit"
6851 };
6852 let tree_note = if progressed {
6853 "changed vs base"
6854 } else {
6855 "unchanged vs base"
6856 };
6857 self.state.event(
6858 "fix",
6859 match &fix.failed {
6860 Some(reason) => {
6866 format!(
6867 "round {round}: fixer's adoption report was lost ({reason}); \
6868 {commit_note}, tree {tree_note}"
6869 )
6870 }
6871 None => format!(
6872 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6873 {tree_note}{}",
6874 fix.addressed.len(),
6875 fix.rejected.len(),
6876 if continuation.outcome == ContinuationOutcome::Resumed {
6877 format!(
6878 " (adoption report recovered after {} continuation(s))",
6879 continuation.attempts
6880 )
6881 } else {
6882 String::new()
6883 },
6884 ),
6885 },
6886 );
6887 round_record.fix = Some(fix);
6888 round_record.progressed = progressed;
6889 Ok(FixStep::Recorded {
6890 report_lost: matches!(
6891 continuation.outcome,
6892 ContinuationOutcome::Exhausted
6893 | ContinuationOutcome::QuotaLost
6894 | ContinuationOutcome::NoSession
6895 ),
6896 })
6897 }
6898
6899 fn record_contested_handoff(&mut self) {
6933 if self.state.contested_handoff.is_some() {
6934 return;
6935 }
6936 let Some(contested) = self
6937 .state
6938 .reviews
6939 .last()
6940 .and_then(ReviewRound::contested_handoff)
6941 else {
6942 return;
6943 };
6944 self.state.event(
6945 "review",
6946 format!(
6947 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6948 wait for the owner's approval",
6949 contested.findings.len(),
6950 contested.rejecters.len()
6951 ),
6952 );
6953 self.state.contested_handoff = Some(contested);
6954 }
6955
6956 async fn stop_unresolved_review(
6957 &mut self,
6958 why: &str,
6959 shell: &[String],
6960 worktree: &Path,
6961 ) -> Result<()> {
6962 self.stop_reviewing_inner(why, shell, worktree, true).await
6963 }
6964
6965 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6966 self.stop_reviewing_inner(why, shell, worktree, false).await
6967 }
6968
6969 async fn stop_reviewing_inner(
6970 &mut self,
6971 why: &str,
6972 shell: &[String],
6973 worktree: &Path,
6974 repairable: bool,
6975 ) -> Result<()> {
6976 let round_idx = self.state.reviews.len() - 1;
6977 let needs_catchup_run = matches!(
6985 self.state.reviews[round_idx].e2e_status(),
6986 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6987 );
6988 if needs_catchup_run {
6989 let round = self.state.reviews[round_idx].round;
6990 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6991 let commands = self.state.config.verify.e2e.clone();
6992 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6993 let cache_dir = self.state.config.cache_dir();
6994 let context = format!(
6995 "round {round}: verification unresolved, catching up before the final decision"
6996 );
6997 let wait = self.repository_wait();
6998 let (outcomes, verify_retried) = with_cache_lease(
6999 &mut self.state,
7000 cache_dir.as_deref(),
7001 "e2e",
7002 "e2e",
7003 worktree,
7004 &attempted_head,
7005 timeout,
7006 &context,
7007 &wait,
7008 |state, budget| {
7009 let shell = shell.to_vec();
7010 let commands = commands.clone();
7011 let context = context.clone();
7012 async move {
7013 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7014 .await
7015 }
7016 },
7017 )
7018 .await?;
7019 let last = &mut self.state.reviews[round_idx];
7020 last.e2e = outcomes;
7021 last.verify_retried = verify_retried;
7022 last.verified_head = Some(attempted_head);
7029 last.verified_at = Some(Timestamp::now());
7030 if verify_inconclusive(&last.e2e) {
7031 self.state.save()?;
7038 return Ok(());
7039 }
7040 last.e2e_deferred = false;
7041 }
7042 let last = &self.state.reviews[round_idx];
7043 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7044
7045 match last.e2e_status() {
7046 E2eStatus::Failed => {
7047 let red: Vec<String> = last
7048 .e2e
7049 .iter()
7050 .filter(|o| !o.ok())
7051 .map(|o| {
7052 format!(
7053 "`{}` -> {:?}\n{}",
7054 o.command,
7055 o.code,
7056 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7057 )
7058 })
7059 .collect();
7060 let can_repair = last.blocking == 0 && last.fix.is_none();
7061 self.state.reviews[round_idx].clean = false;
7062 self.state
7063 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7064 self.state.status = RunStatus::Blocked;
7065 if repairable
7066 && can_repair
7067 && self.state.reviews.len() < self.state.config.graph.review_rounds
7068 {
7069 self.state.status = RunStatus::Reviewing;
7070 }
7071 }
7072 E2eStatus::ResourceBlocked => {
7077 self.state.event(
7078 "review",
7079 format!(
7080 "{why}; e2e could not run (resource unavailable); not \
7081 deciding yet"
7082 ),
7083 );
7084 }
7085 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7086 self.state.event(
7087 "review",
7088 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7089 );
7090 self.record_contested_handoff();
7091 self.state.status = RunStatus::Gating;
7092 }
7093 }
7094 self.state.save()?;
7095 Ok(())
7096 }
7097
7098 async fn gate(&mut self) -> Result<()> {
7101 if self.state.status == RunStatus::Failed
7113 || self
7114 .state
7115 .base_sync
7116 .as_ref()
7117 .is_some_and(|s| s.conflict.is_some())
7118 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7119 != Some(RunStatus::Gating)
7120 {
7121 return Ok(());
7122 }
7123 if self.state.gate_ran {
7124 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7135 self.state.status = RunStatus::Blocked;
7136 self.state.save()?;
7137 }
7138 return Ok(());
7139 }
7140 let Some(winner) = self.state.winner().cloned() else {
7141 return Ok(());
7142 };
7143 self.state.status = RunStatus::Gating;
7144 let mut outcomes = self.run_gate(&winner).await?;
7145 loop {
7146 if verify_inconclusive(&outcomes) {
7157 self.state.save()?;
7158 return Ok(());
7159 }
7160 if outcomes.iter().all(CommandOutcome::ok) {
7161 break;
7162 }
7163 match self.gate_fix_round(&winner, &outcomes).await? {
7164 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7165 GateFix::Stop => break,
7166 GateFix::Defer => {
7167 self.state.save()?;
7168 return Ok(());
7169 }
7170 }
7171 }
7172 let passed = outcomes.iter().all(CommandOutcome::ok);
7173 self.state.gate = outcomes;
7174 self.state.gate_ran = true;
7175 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7176 if !passed {
7177 self.state.status = RunStatus::Blocked;
7178 let spent = self.state.gate_fixes.len();
7179 self.state.event(
7180 "gate",
7181 if spent == 0 {
7182 "gate failed; not merging".to_owned()
7183 } else {
7184 format!("gate failed after {spent} gate-fix round(s); not merging")
7185 },
7186 );
7187 }
7188 self.state.save()?;
7189 Ok(())
7190 }
7191
7192 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7202 let commands = state.config.verify.pre_gate.clone();
7203 if commands.is_empty() {
7204 return;
7205 }
7206 let shell = state.config.shell();
7207 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7208 let (outcomes, _) = run_commands(
7209 state,
7210 "pre_gate",
7211 "pre_gate",
7212 0,
7213 &shell,
7214 &commands,
7215 &winner.worktree,
7216 timeout,
7217 )
7218 .await;
7219 for o in &outcomes {
7220 if !o.ok() {
7221 tracing::warn!(
7222 "pre_gate `{}` failed ({:?}); the gate decides",
7223 o.command,
7224 o.code
7225 );
7226 }
7227 state.event(
7228 "pre_gate",
7229 format!(
7230 "`{}` -> {}",
7231 o.command,
7232 if o.ok() {
7233 "pass".to_owned()
7234 } else {
7235 format!(
7236 "FAIL ({:?})\n{}",
7237 o.code,
7238 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7239 )
7240 }
7241 ),
7242 );
7243 }
7244 state.pre_gate = outcomes;
7245 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7246 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7247 Ok(head) => {
7248 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7249 state.pre_gate_commit = Some(head);
7250 }
7251 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7252 },
7253 Ok(false) => {}
7254 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7255 }
7256 if let Err(e) = state.save() {
7257 tracing::warn!("could not persist the pre_gate record: {e:#}");
7258 }
7259 }
7260
7261 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7264 let wait = self.repository_wait();
7265 run_landing_gate(&mut self.state, winner, &wait).await
7266 }
7267
7268 async fn gate_fix_round(
7281 &mut self,
7282 winner: &Candidate,
7283 outcomes: &[CommandOutcome],
7284 ) -> Result<GateFix> {
7285 let cap = self.state.config.graph.gate_fix_rounds;
7286 let spent = self.state.gate_fixes.len();
7287 if spent >= cap {
7288 if cap > 0 {
7289 self.state.event(
7290 "gate",
7291 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7292 );
7293 }
7294 return Ok(GateFix::Stop);
7295 }
7296 if !gate_fixable(outcomes) {
7297 self.state.event(
7298 "gate",
7299 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7300 command or similar); not spending a fix round on it",
7301 );
7302 return Ok(GateFix::Stop);
7303 }
7304 let min_free = self.state.config.disk.min_free_bytes;
7305 if min_free > 0 {
7306 match crate::disk::free_bytes(&winner.worktree) {
7307 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7308 Ok(free) => {
7309 self.state.event(
7310 "gate",
7311 format!(
7312 "only {free} bytes free ({min_free} required by `[disk] \
7313 min_free_bytes`); not spending a fix round on a failure the disk \
7314 may explain"
7315 ),
7316 );
7317 return Ok(GateFix::Stop);
7318 }
7319 Err(e) => {
7320 self.state.event(
7321 "gate",
7322 format!("free disk space could not be measured ({e:#}); no fix round"),
7323 );
7324 return Ok(GateFix::Stop);
7325 }
7326 }
7327 }
7328
7329 let attempt = spent + 1;
7330 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7331 let base = self.landing_base();
7332 let fix_prompt = prompt::gate_fix_in(
7333 &self.state.instruction,
7334 &failed,
7335 attempt,
7336 cap,
7337 &self.state.config.graph.language,
7338 &self.state.config.graph.github_language,
7339 );
7340 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7341 let sessions = self.state.config.graph.sessions;
7342 let artifacts = self.merge_repair_artifacts();
7343 self.state.event(
7344 "gate",
7345 format!("gate failed; gate-fix round {attempt} of {cap}"),
7346 );
7347 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7348 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7349 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7352 head: before.clone(),
7353 passed: false,
7354 outcomes: Vec::new(),
7355 });
7356 self.state.save()?;
7357 let (_, seat, out) = self
7358 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7359 prompt: fix_prompt.clone(),
7360 spec,
7361 seat,
7362 cwd: winner.worktree.clone(),
7363 timeout,
7364 allow_write: true,
7365 sessions,
7366 artifacts: artifacts.clone(),
7367 stem: format!("gate-fix-{attempt}"),
7368 handover: None,
7369 })
7370 .await;
7371 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7374 return Ok(GateFix::Defer);
7375 }
7376 let mut record = GateFixRecord {
7377 agent: seat.agent.clone(),
7378 failed,
7379 notes: String::new(),
7380 committed: false,
7381 error: None,
7382 };
7383 match out {
7384 AgentOutcome::Ok(o) => {
7385 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7388 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7389 }
7390 }
7391 AgentOutcome::Dropped(_) => {
7392 record.error = Some("the CLI dropped the stream".to_owned());
7393 }
7394 AgentOutcome::Quota(o) => {
7395 self.state.quota.push(QuotaLoss {
7396 seat: seat.key.clone(),
7397 node: "gate-fix".to_owned(),
7398 at: Timestamp::now(),
7399 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7400 });
7401 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7402 }
7403 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7404 }
7405 self.state.seats.insert(seat.key.clone(), seat);
7406 let coordination = self.coordinate(&winner.worktree).await?;
7407 if let Ok(r) = git::rescue_commit(
7408 &winner.worktree,
7409 &format!("magi: gate fix {attempt} (uncommitted work)"),
7410 )
7411 .await
7412 {
7413 self.state.note_withheld("gate-fix", &r.withheld);
7414 }
7415 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7416 record.committed = after != before;
7417 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7418 let note = record.error.clone();
7419 self.state.gate_fixes.push(record);
7420 self.state.save()?;
7421 if !changed {
7422 self.state.event(
7423 "gate",
7424 match note {
7425 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7426 None => format!("gate-fix round {attempt}: the tree did not change"),
7427 },
7428 );
7429 return Ok(GateFix::Stop);
7430 }
7431 self.state.event(
7432 "gate",
7433 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7434 );
7435
7436 drop(coordination);
7437 let wait = self.repository_wait();
7438 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7439 }
7440
7441 async fn guarded_pr_message(
7448 &mut self,
7449 winner: &Candidate,
7450 facts: Option<&BranchFacts>,
7451 posting: bool,
7452 ) -> Result<Option<PrMessage>> {
7453 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7454 let head = match final_head.as_deref() {
7455 Some(h) => crate::pr_outcome::FinalHead::At(h),
7456 None => crate::pr_outcome::FinalHead::Unknown,
7457 };
7458 let summarized = if posting {
7460 Some(self.summarize_change(winner, final_head.as_deref()).await)
7461 } else {
7462 None
7463 };
7464 if self.state.resource_block.is_some() {
7469 self.state.event(
7470 "pr-summary",
7471 "a shared resource was busy; nothing was published, resume to summarize again",
7472 );
7473 return Ok(None);
7474 }
7475 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7476 anyhow::bail!(
7477 "the final commit moved during public description generation; retry before publishing"
7478 );
7479 }
7480 let note = match (&summarized, &self.state.change_summary) {
7481 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7482 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7483 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7484 (None, _) => ChangeNote::NotAsked,
7485 };
7486 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7487 if !posting {
7488 let identity = crate::scrub::Identity::current();
7489 return Ok(Some(PrMessage {
7490 title: crate::scrub::scrub(&pr.title, &identity),
7491 body: crate::scrub::scrub(&pr.body, &identity),
7492 }));
7493 }
7494 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7497 if let Some(known) = self
7498 .state
7499 .github_text
7500 .clone()
7501 .filter(|g| g.fingerprint == raw_fp)
7502 {
7503 return self.settle_withheld(winner, &pr, known).await;
7504 }
7505 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7506 let mut violations =
7507 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7508 let identity = crate::scrub::Identity::current();
7509 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7510 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7511 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7512 {
7513 violations.push(crate::github_text::Violation::SensitiveData);
7514 }
7515 let has_language = violations
7519 .iter()
7520 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7521 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7522 self.state.event(
7523 "github-text",
7524 if has_language {
7525 format!("description rejected: {violations:?}; requesting one rewrite")
7526 } else {
7527 format!("description rejected: {violations:?}; asking the owner")
7528 },
7529 );
7530 let mut rewritten = false;
7531 let mut rejected_rewrite = None;
7532 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7533 {
7534 let output = text;
7535 if let Some(title) = summary_title(&output) {
7536 let raw_body = summary_without_title(&output);
7539 let clean_title = crate::scrub::scrub(&title, &identity);
7540 let mut body = crate::scrub::scrub(&raw_body, &identity);
7541 let redacted = clean_title != title || body != raw_body;
7542 let title = clean_title;
7543 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7544 if !complete {
7545 self.state.event(
7546 "github-text",
7547 "description rewrite omitted required findings",
7548 );
7549 }
7550 if !body.trim().is_empty() {
7551 body.push_str("\n\n");
7552 body.push_str(&public_pr_evidence(&self.state, head));
7553 }
7554 let marker = format!("magi:run/{}", self.state.id);
7555 if !body.trim().is_empty() && !body.contains(&marker) {
7556 body.push_str(&format!("\n\n{marker}"));
7557 }
7558 let rewrite = PrMessage {
7559 title: title.clone(),
7560 body: body.clone(),
7561 };
7562 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7563 let mut failed = crate::github_text::check_for(
7564 &self.state.config.graph,
7565 &title,
7566 &body,
7567 redecision,
7568 );
7569 if (!crate::github_text::shareable(&title)
7570 || !crate::github_text::shareable(&body))
7571 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7572 {
7573 failed.push(crate::github_text::Violation::SensitiveData);
7574 }
7575 if complete
7576 && !title.trim().is_empty()
7577 && !body.trim().is_empty()
7578 && !failed.is_empty()
7579 {
7580 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7581 title: title.clone(),
7582 body: body.clone(),
7583 failed: failed
7584 .iter()
7585 .map(|v| crate::github_text::category(*v).to_owned())
7586 .collect(),
7587 });
7588 }
7589 if complete
7590 && !title.trim().is_empty()
7591 && !raw_body.trim().is_empty()
7592 && failed.is_empty()
7593 {
7594 decision = redecision;
7595 pr = PrMessage { title, body };
7596 rewritten = true;
7597 self.state
7598 .event("github-text", "description rewrite passed");
7599 if redacted {
7600 self.state.event(
7601 "github-text",
7602 "sensitive data removed from description rewrite",
7603 );
7604 }
7605 }
7606 }
7607 }
7608 if !rewritten {
7609 self.state.event(
7610 "github-text",
7611 "description rewrite unavailable or rejected; asking the owner",
7612 );
7613 if let Some(w) =
7614 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7615 {
7616 self.retire_github_text_question("the withheld text changed");
7619 let record = crate::run::GithubTextAsk {
7620 rewrite: rejected_rewrite,
7621 fingerprint: raw_fp,
7622 title: w.title,
7623 body: w.body,
7624 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7625 question: None,
7626 asks: 0,
7627 resolved: false,
7628 chosen_title: None,
7629 choice: None,
7630 };
7631 return self.settle_withheld(winner, &pr, record).await;
7632 }
7633 }
7634 }
7635 let (title, body) =
7636 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7637 Ok(Some(PrMessage { title, body }))
7638 }
7639
7640 async fn summarize_change(
7651 &mut self,
7652 winner: &Candidate,
7653 head: Option<&str>,
7654 ) -> std::result::Result<(), String> {
7655 let Some(head) = head else {
7656 return Err("the final commit could not be read".to_owned());
7657 };
7658 let remote = self.state.config.merge.remote.clone();
7659 let tracking = format!("{remote}/{}", self.state.base_branch);
7660 if !git::rev_exists(&winner.worktree, &tracking).await {
7661 return Err(format!("the remote base `{tracking}` could not be read"));
7662 }
7663 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7664 Ok(b) if !b.is_empty() => b,
7665 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7666 };
7667 let evidence = crate::pr_outcome::summary_evidence(
7668 &self.state,
7669 crate::pr_outcome::FinalHead::At(head),
7670 );
7671 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7672 if self
7673 .state
7674 .change_summary
7675 .as_ref()
7676 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7677 {
7678 return Ok(());
7679 }
7680 if self.state.change_summary.take().is_some() {
7681 self.state.event(
7682 "pr-summary",
7683 "the final diff or evidence changed; the earlier change summary was discarded",
7684 );
7685 }
7686 if let Err(e) = write_artifact(
7688 &self.state,
7689 &format!("public-description-{fingerprint}.md"),
7690 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7691 ) {
7692 tracing::warn!("could not save public description evidence: {e:#}");
7693 }
7694 let chain = match agent::pick_chain(
7695 &self.state.config.agents,
7696 self.state.config.roles.synthesizer.as_ref(),
7697 &agent::installed,
7698 "synthesizer",
7699 ) {
7700 Ok(chain) => chain,
7701 Err(e) => {
7702 tracing::warn!("no agent for the change summary: {e:#}");
7703 self.state.event("pr-summary", "no summary agent available");
7704 return Err("no agent was available to summarize it".to_owned());
7705 }
7706 };
7707 let subjects = git::subjects(&winner.worktree, &base, head)
7708 .await
7709 .unwrap_or_default();
7710 let stat = git::diff_stat(&winner.worktree, &base, head)
7711 .await
7712 .unwrap_or_default();
7713 let stat: String = stat.chars().take(6_000).collect();
7714 let mut prompt = crate::prompt::change_summary_in(
7715 &base,
7716 head,
7717 &subjects,
7718 stat.trim(),
7719 &self.state.config.graph.github_language,
7720 );
7721 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7722 let sessions = self.state.config.graph.sessions;
7723 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7724 let prompts = self.state.config.prompts.clone();
7725 let cache = self.state.config.cache_dir();
7726 let run = self.state.id.clone();
7727 for (i, spec) in chain.into_iter().enumerate() {
7728 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7729 let github_spec = match agent::for_github_text(
7730 &spec,
7731 &self.state.config.graph.github_language,
7732 &winner.worktree,
7733 ) {
7734 Ok(spec) => spec,
7735 Err(e) => {
7736 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7737 continue;
7738 }
7739 };
7740 let job = SeatJob {
7741 spec: github_spec,
7742 seat,
7743 cwd: winner.worktree.clone(),
7744 prompt: prompt.clone(),
7745 timeout,
7746 allow_write: false,
7747 sessions,
7748 artifacts: agent::artifacts_dir(&self.state.dir()),
7749 stem: if i == 0 {
7750 "change-summary".to_owned()
7751 } else {
7752 format!("change-summary-{}", spec.id)
7753 },
7754 handover: None,
7755 };
7756 let ctx = WaveCtx {
7757 run: &run,
7758 node: "pr-summary",
7759 prompts: &prompts,
7760 cache: cache.as_deref(),
7761 round: None,
7762 carry_seats: false,
7763 pauses: Some((&self.pause, &self.interrupt)),
7764 };
7765 let (seat, outcome) =
7766 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7767 self.state.seats.insert(seat.key.clone(), seat);
7768 let text = match outcome {
7769 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7770 AgentOutcome::ResourceBlocked(why) => {
7771 if self.state.resource_block.is_none() {
7774 self.state.resource_block = Some(crate::run::ResourceBlock {
7775 cause: crate::run::ResourceBlockCause::Other,
7776 node: "pr-summary".to_owned(),
7777 kind: "shared resource".to_owned(),
7778 detail: why,
7779 at: Timestamp::now(),
7780 });
7781 }
7782 return Err("a shared resource was busy".to_owned());
7783 }
7784 _ => continue,
7785 };
7786 let blind = &self.state.config.blind;
7787 let Some(title) = summary_title(&text) else {
7788 continue;
7789 };
7790 let body: String = summary_without_title(&text)
7791 .lines()
7792 .filter(|l| !l.contains("magi:run/"))
7793 .collect::<Vec<_>>()
7794 .join("\n");
7795 let body = blind::sanitize_prose(body.trim(), blind);
7796 let title = blind::sanitize_prose(&title, blind);
7797 if body.trim().is_empty() || title.trim().is_empty() {
7798 continue;
7799 }
7800 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7801 continue;
7802 }
7803 if git::rev_parse(&winner.worktree, "HEAD")
7805 .await
7806 .ok()
7807 .as_deref()
7808 != Some(head)
7809 {
7810 return Err("the branch moved while the summary was written".to_owned());
7811 }
7812 if git::merge_base(&winner.worktree, &tracking, head)
7813 .await
7814 .ok()
7815 .as_deref()
7816 != Some(base.as_str())
7817 || crate::pr_outcome::summary_evidence(
7818 &self.state,
7819 crate::pr_outcome::FinalHead::At(head),
7820 ) != evidence
7821 {
7822 return Err("the summary evidence changed while it was written".to_owned());
7823 }
7824 self.state.change_summary = Some(crate::run::ChangeSummary {
7825 head: head.to_owned(),
7826 base,
7827 evidence: fingerprint,
7828 title,
7829 body,
7830 });
7831 self.state.event(
7832 "pr-summary",
7833 format!("change summary written by {} for the final diff", spec.id),
7834 );
7835 if let Err(e) = self.state.save() {
7836 tracing::warn!("could not save the change summary: {e:#}");
7837 }
7838 return Ok(());
7839 }
7840 self.state
7841 .event("pr-summary", "no agent produced a change summary");
7842 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7843 }
7844
7845 async fn rewrite_pr_text(
7855 &mut self,
7856 winner: &Candidate,
7857 pr: &PrMessage,
7858 violations: &[crate::github_text::Violation],
7859 ) -> Option<String> {
7860 let prompt = format!(
7861 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7862 pr.title, pr.body
7863 );
7864 let sessions = self.state.config.graph.sessions;
7865 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7866 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7869 let key = format!("impl-{}", winner.label);
7870 if let Some(spec) = self
7871 .state
7872 .config
7873 .agents
7874 .iter()
7875 .find(|a| a.id == winner.agent)
7876 .cloned()
7877 && let Some(existing) = self.state.seats.get(&key)
7878 && existing.agent == spec.id
7879 && agent::has_session(spec.kind, existing, sessions)
7880 {
7881 attempts.push((spec, Some(existing.clone())));
7882 }
7883 match agent::pick_chain(
7884 &self.state.config.agents,
7885 self.state.config.roles.language_judge.as_ref(),
7886 &agent::installed,
7887 "language_judge",
7888 ) {
7889 Ok(chain) => {
7890 for spec in chain {
7891 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7892 continue;
7893 }
7894 attempts.push((spec, None));
7895 }
7896 }
7897 Err(e) if attempts.is_empty() => {
7898 tracing::warn!("no agent for the description rewrite: {e:#}");
7899 self.state
7900 .event("github-text", "no rewrite agent available");
7901 return None;
7902 }
7903 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7904 }
7905 let prompts = self.state.config.prompts.clone();
7906 let cache = self.state.config.cache_dir();
7907 let run = self.state.id.clone();
7908 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7909 let resumed = resumed_seat.is_some();
7910 let seat = resumed_seat.unwrap_or_else(|| {
7911 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7912 });
7913 let github_spec = match agent::for_github_text(
7914 &spec,
7915 &self.state.config.graph.github_language,
7916 &winner.worktree,
7917 ) {
7918 Ok(spec) => spec,
7919 Err(e) => {
7920 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7921 continue;
7922 }
7923 };
7924 let job = SeatJob {
7925 spec: github_spec,
7926 seat,
7927 cwd: winner.worktree.clone(),
7928 prompt: prompt.clone(),
7929 timeout: retry_budget(timeout_full, resumed),
7930 allow_write: false,
7931 sessions,
7932 artifacts: agent::artifacts_dir(&self.state.dir()),
7933 stem: if i == 0 {
7934 "github-text-rewrite".to_owned()
7935 } else {
7936 format!("github-text-rewrite-{}", spec.id)
7937 },
7938 handover: None,
7939 };
7940 let ctx = WaveCtx {
7941 run: &run,
7942 node: "github-text",
7943 prompts: &prompts,
7944 cache: cache.as_deref(),
7945 round: None,
7946 carry_seats: false,
7947 pauses: Some((&self.pause, &self.interrupt)),
7948 };
7949 let (seat, outcome) =
7950 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7951 self.state.seats.insert(seat.key.clone(), seat);
7952 match outcome {
7953 AgentOutcome::Ok(o)
7954 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7955 {
7956 return Some(o.text);
7957 }
7958 AgentOutcome::ResourceBlocked(_) => return None,
7959 _ => {}
7960 }
7961 }
7962 None
7963 }
7964
7965 fn retire_github_text_question(&mut self, why: &str) {
7967 let Some(id) = self
7968 .state
7969 .github_text
7970 .as_ref()
7971 .and_then(|g| g.question.clone())
7972 else {
7973 return;
7974 };
7975 let _ = ask::Questions::open().update(&id, |q| {
7976 q.abandon(why);
7977 Ok(())
7978 });
7979 }
7980
7981 async fn settle_withheld(
7985 &mut self,
7986 winner: &Candidate,
7987 pr: &PrMessage,
7988 mut g: crate::run::GithubTextAsk,
7989 ) -> Result<Option<PrMessage>> {
7990 use crate::github_text as gt;
7991 let store = ask::Questions::open();
7992 let timeout = self.state.config.graph.answer_timeout;
7993 if !g.resolved {
7994 let mut standing = g
7995 .question
7996 .as_ref()
7997 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7998 if let Some(q) = standing.as_ref()
7999 && q.status.open()
8000 && gt::expired(q, timeout)
8001 {
8002 let why = format!("no answer within {}s of asking", timeout.max(1));
8003 standing = store
8004 .update(&q.id, |q| {
8005 q.abandon(&why);
8006 Ok(())
8007 })
8008 .ok()
8009 .map(|(q, ())| q);
8010 }
8011 let mut ask_again: Option<Option<Vec<String>>> = None;
8012 match standing {
8013 None if g.asks <= 1 => {
8016 g.asks = 0;
8017 ask_again = Some(None);
8018 }
8019 None => {
8020 g.resolved = true;
8021 g.chosen_title = None;
8022 g.choice = Some(GithubTextChoice::Fallback);
8023 }
8024 Some(q) if q.status.open() => {
8025 self.park_on_github_text(&g)?;
8026 return Ok(None);
8027 }
8028 Some(q) => match gt::read_reply(&q) {
8029 gt::Reply::Fallback => {
8030 g.resolved = true;
8031 g.chosen_title = None;
8032 g.choice = Some(GithubTextChoice::Fallback);
8033 }
8034 gt::Reply::Original => {
8035 g.resolved = true;
8036 g.chosen_title = None;
8037 g.choice = Some(GithubTextChoice::Original);
8038 }
8039 gt::Reply::Rewrite => {
8040 g.resolved = true;
8041 g.chosen_title = None;
8042 g.choice = Some(if g.rewrite.is_some() {
8043 GithubTextChoice::Rewrite
8044 } else {
8045 GithubTextChoice::Fallback
8046 });
8047 }
8048 gt::Reply::Title(text) => {
8049 let first = text
8050 .lines()
8051 .map(str::trim)
8052 .find(|l| !l.is_empty())
8053 .unwrap_or("");
8054 let decision = if first.is_empty() {
8055 None
8056 } else {
8057 let candidate = PrMessage {
8058 title: first.to_owned(),
8059 body: String::new(),
8060 };
8061 self.judge_pr_language(&winner.worktree, &candidate).await
8062 };
8063 match gt::vet_title(
8064 &text,
8065 decision,
8066 self.state.config.graph.github_english(),
8067 ) {
8068 Ok(title) => {
8069 g.resolved = true;
8070 g.chosen_title = Some(title);
8071 g.choice = Some(GithubTextChoice::MyTitle);
8072 }
8073 Err(cats) if g.asks < 2 => {
8074 ask_again = Some(Some(
8075 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8076 ))
8077 }
8078 Err(_) => {
8079 g.resolved = true;
8080 g.chosen_title = None;
8081 g.choice = Some(GithubTextChoice::Fallback);
8082 }
8083 }
8084 }
8085 },
8086 }
8087 if let Some(rejected) = ask_again {
8088 let w = gt::Withheld {
8091 title: g.title,
8092 body: g.body,
8093 categories: g.categories.clone(),
8094 };
8095 let artifact = match crate::run::write_artifact(
8098 &self.state,
8099 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8100 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8101 ) {
8102 Ok(p) => Some(p),
8103 Err(e) => {
8104 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8105 None
8106 }
8107 };
8108 let (detail, comparison) = gt::question_content(
8109 &w,
8110 &pr.title,
8111 &pr.body,
8112 &self.state.id,
8113 rejected.as_deref(),
8114 artifact.as_deref(),
8115 g.rewrite.as_ref(),
8116 );
8117 let mut q = ask::Question::new(
8118 self.state.id.clone(),
8119 gt::ASK_NODE.to_owned(),
8120 gt::ASK_SEAT.to_owned(),
8121 gt::question_summary(&self.state.config.graph.language, &w),
8122 detail,
8123 gt::question_choices(&w, g.rewrite.as_ref()),
8124 );
8125 q.posting_comparison = Some(comparison);
8126 q.answer_timeout = timeout;
8127 g.question = Some(q.id.clone());
8130 g.asks += 1;
8131 self.state.github_text = Some(g.clone());
8132 self.state.event(
8133 "github-text",
8134 format!("asking the owner about the withheld text ({})", q.short()),
8135 );
8136 self.state.save()?;
8137 store
8138 .put_executable(&mut q)
8139 .context("file the posting-gate question")?;
8140 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8141 tracing::warn!(
8142 "could not notify about posting-gate question {}: {e:#}",
8143 q.short()
8144 );
8145 }
8146 self.park_on_github_text(&g)?;
8147 return Ok(None);
8148 }
8149 }
8150 let resolved = g.chosen_title.clone();
8151 let original = g.choice == Some(GithubTextChoice::Original);
8152 self.state.event(
8154 "github-text",
8155 if original {
8156 format!(
8157 "posting the original text for: {}",
8158 match (g.title, g.body) {
8159 (true, true) => "title, description",
8160 (true, false) => "title",
8161 _ => "description",
8162 }
8163 )
8164 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8165 "posting the saved rewrite title and description".to_owned()
8166 } else if resolved.is_some() {
8167 "posting the owner's replacement title".to_owned()
8168 } else {
8169 "posting the neutral text for the withheld fields".to_owned()
8170 },
8171 );
8172 self.state.github_text = Some(g.clone());
8173 self.state.save()?;
8174 if g.choice == Some(GithubTextChoice::Rewrite)
8175 && let Some(r) = g.rewrite.as_ref()
8176 {
8177 return Ok(Some(PrMessage {
8178 title: r.title.clone(),
8179 body: r.body.clone(),
8180 }));
8181 }
8182 let w = gt::Withheld {
8183 title: g.title,
8184 body: g.body,
8185 categories: g.categories.clone(),
8186 };
8187 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8188 if let Some(t) = resolved {
8189 title = t;
8190 }
8191 let english = gt::LanguageDecision {
8193 title_english: true,
8194 body_english: true,
8195 body_complete: true,
8196 };
8197 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8198 if original {
8199 if g.title {
8202 title = pr.title.clone();
8203 }
8204 if g.body {
8205 body = pr.body.clone();
8206 }
8207 }
8208 Ok(Some(PrMessage { title, body }))
8209 }
8210
8211 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8212 self.state.github_text = Some(g.clone());
8213 self.state.end_pass_parked(ParkCause::Question);
8214 self.state.parked = true;
8215 self.state.event(
8216 "github-text",
8217 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8218 );
8219 self.state.save()
8220 }
8221
8222 async fn judge_pr_language(
8225 &mut self,
8226 cwd: &Path,
8227 pr: &PrMessage,
8228 ) -> Option<crate::github_text::LanguageDecision> {
8229 self.state.config.roles.language_judge.as_ref()?;
8230 if !self.state.config.graph.language_gate() {
8231 return None;
8232 }
8233 let decision =
8234 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8235 self.state.event(
8236 "github-text",
8237 if decision.is_some() {
8238 "language decided by the language judge"
8239 } else {
8240 "language judge unavailable; using built-in heuristics"
8241 },
8242 );
8243 decision
8244 }
8245
8246 async fn merge(&mut self) -> Result<()> {
8247 if self
8262 .state
8263 .base_sync
8264 .as_ref()
8265 .is_some_and(|s| s.conflict.is_some())
8266 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8267 != Some(RunStatus::Gating)
8268 || !self.state.gate_allows_progress()
8276 {
8277 return Ok(());
8278 }
8279 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8288 return Ok(());
8289 }
8290 let Some(winner) = self.state.winner().cloned() else {
8291 return Ok(());
8292 };
8293 let repo = self.state.repo.clone();
8294 let base = self.state.base_branch.clone();
8295 let mode = self.state.config.merge.mode;
8296 let style = self.state.config.merge.style;
8297 let facts = if is_review_run(&self.state) {
8298 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8299 let start = review_base(
8300 &repo,
8301 &self.state.config.merge.remote,
8302 &base,
8303 &self.state.base_commit,
8304 &winner.branch,
8305 )
8306 .await;
8307 branch_facts(&repo, &start, &winner.branch).await
8308 } else {
8309 None
8310 };
8311 let leaked = if is_review_run(&self.state) {
8314 leaked_subjects(&self.state, &winner.branch).await
8315 } else {
8316 Some(Vec::new())
8317 };
8318 let Some(pr) = self
8322 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8323 .await?
8324 else {
8325 return Ok(());
8326 };
8327 let message = pr.commit_message();
8328
8329 let outcome = match mode {
8330 MergeMode::None => MergeOutcome {
8331 mode,
8332 ok: true,
8333 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8334 empty: false,
8335 },
8336 MergeMode::Pr | MergeMode::Local
8337 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8338 {
8339 MergeOutcome {
8340 mode,
8341 ok: false,
8342 detail: empty_candidate_detail(&self.state, &base),
8343 empty: true,
8344 }
8345 }
8346 MergeMode::Local => {
8347 let on = git::current_branch(&repo).await?;
8348 if on.as_deref() != Some(base.as_str()) {
8349 MergeOutcome {
8350 mode,
8351 ok: false,
8352 detail: format!(
8353 "{} has {} checked out, not the base branch {base}",
8354 repo.display(),
8355 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8356 ),
8357 empty: false,
8358 }
8359 } else if !git::is_clean(&repo).await? {
8360 MergeOutcome {
8361 mode,
8362 ok: false,
8363 detail: format!("{} is dirty; refusing to merge", repo.display()),
8364 empty: false,
8365 }
8366 } else {
8367 let out = match style {
8368 MergeStyle::Merge => {
8369 git::merge_no_ff(&repo, &winner.branch, &message).await?
8370 }
8371 MergeStyle::Squash => {
8372 git::merge_squash(&repo, &winner.branch, &message).await?
8373 }
8374 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8375 };
8376 MergeOutcome {
8377 mode,
8378 ok: out.ok(),
8379 detail: if out.ok() { out.stdout } else { out.stderr },
8380 empty: false,
8381 }
8382 }
8383 }
8384 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8385 self.state.pr_publication_pending = true;
8386 self.state.save()?;
8387 let out =
8388 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8389 self.state.pr_publication_pending = out.is_err();
8390 MergeOutcome {
8391 mode,
8392 ok: out.is_ok(),
8393 detail: match out {
8394 Ok(url) => url,
8395 Err(e) => format!("{e:#}"),
8396 },
8397 empty: false,
8398 }
8399 }
8400 MergeMode::Pr => {
8401 let remote = self.state.config.merge.remote.clone();
8402 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8403 if !pushed.ok() {
8404 MergeOutcome {
8405 mode,
8406 ok: false,
8407 detail: pushed.stderr,
8408 empty: false,
8409 }
8410 } else {
8411 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8417 let out = match pr_merge_plan(found) {
8418 PrPlan::Create => {
8419 gh_pr_create(
8420 &winner.worktree,
8421 &base,
8422 &winner.branch,
8423 &pr.title,
8424 &pr.body,
8425 )
8426 .await
8427 }
8428 PrPlan::Adopt { url, title } => {
8429 self.state
8430 .event("merge", format!("Pr: adopted open pull request {url}"));
8431 if self.state.github_text.as_ref().is_some_and(|g| {
8432 g.resolved
8433 && g.choice == Some(GithubTextChoice::Rewrite)
8434 && g.rewrite.is_some()
8435 }) {
8436 land::set_pr_text_verbatim(
8439 &winner.worktree,
8440 &url,
8441 &pr.title,
8442 &pr.body,
8443 )
8444 .await
8445 .map(|()| url)
8446 } else {
8447 if title != pr.title
8448 && (!is_review_run(&self.state)
8449 || leaked
8450 .as_deref()
8451 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8452 && let Err(e) =
8453 if self.state.github_text.as_ref().is_some_and(|g| {
8454 g.resolved
8455 && g.title
8456 && g.choice == Some(GithubTextChoice::Original)
8457 }) {
8458 land::set_pr_title_verbatim(
8460 &winner.worktree,
8461 &url,
8462 &pr.title,
8463 )
8464 .await
8465 } else {
8466 land::set_pr_title(
8467 &mut self.state,
8468 &winner.worktree,
8469 &url,
8470 &pr.title,
8471 )
8472 .await
8473 }
8474 {
8475 tracing::warn!("could not refresh title of {url}: {e:#}");
8476 self.state
8477 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8478 }
8479 Ok(url)
8480 }
8481 }
8482 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8483 };
8484 match out {
8485 Ok(url) => MergeOutcome {
8486 mode,
8487 ok: true,
8488 detail: url,
8489 empty: false,
8490 },
8491 Err(e) => MergeOutcome {
8492 mode,
8493 ok: false,
8494 detail: e.to_string(),
8495 empty: false,
8496 },
8497 }
8498 }
8499 }
8500 };
8501
8502 self.state.status = match (mode, outcome.ok) {
8503 (MergeMode::None, _) => RunStatus::Ready,
8504 (_, true) if outcome.empty => RunStatus::Ready,
8505 (MergeMode::Pr, true) => RunStatus::Ready,
8506 (_, true) => RunStatus::Merged,
8507 (_, false) => RunStatus::Blocked,
8508 };
8509 self.state.event(
8510 "merge",
8511 format!(
8512 "{:?}: {}",
8513 mode,
8514 outcome.detail.lines().next().unwrap_or("")
8515 ),
8516 );
8517 self.state.merge = Some(outcome);
8518 self.state.save()?;
8519
8520 if self.state.config.graph.land
8526 && mode == MergeMode::Pr
8527 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8528 {
8529 self.run_land().await?;
8530 }
8531 self.settle_questions();
8536 Ok(())
8537 }
8538
8539 async fn run_land(&mut self) -> Result<()> {
8550 crate::pr_head::ensure_owner(&self.state)?;
8551 let url = self
8552 .state
8553 .pr
8554 .as_ref()
8555 .map(|pr| pr.url.clone())
8556 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8557 .unwrap_or_default();
8558 let url = url.lines().next().unwrap_or("").trim().to_owned();
8559 if !url.starts_with("http") {
8560 return Ok(());
8561 }
8562 loop {
8565 match land::land(&mut self.state, &url).await {
8566 Ok(_)
8567 if self
8568 .state
8569 .merge_repairs
8570 .last()
8571 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8572 {
8573 if self.repair_merge_review().await? {
8574 continue;
8575 }
8576 }
8577 Ok(pr) if self.state.parked => {
8578 let _ = pr;
8582 }
8583 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8584 self.state.status = RunStatus::Superseded;
8585 self.state.save()?;
8586 }
8587 Ok(pr) => {
8588 self.state.status = match pr.state {
8589 land::PrLifecycle::Merged => RunStatus::Merged,
8590 _ => RunStatus::Blocked,
8591 };
8592 if bump::should_release_bump(self.state.status)
8599 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8600 {
8601 self.state
8607 .event("bump", format!("release bump skipped: {e:#}"));
8608 }
8609 if self.state.status == RunStatus::Merged {
8612 crate::followup::after_merge(&mut self.state, &pr.url).await;
8613 }
8614 self.state.save()?;
8615 }
8616 Err(e) => {
8617 if e.is::<crate::repo_lock::ResourceBlocked>() {
8618 return Err(e);
8619 }
8620 self.state.status = RunStatus::Blocked;
8621 self.state.event("land", format!("gave up: {e}"));
8622 self.state.save()?;
8623 }
8624 }
8625 break;
8626 }
8627 Ok(())
8628 }
8629
8630 async fn repair_merge_review(&mut self) -> Result<bool> {
8633 use crate::run::MergeRepairPhase as Phase;
8634 crate::pr_head::ensure_owner(&self.state)?;
8635 let i = self.state.merge_repairs.len() - 1;
8636 let request = self.state.merge_repairs[i].clone();
8637 if request.phase == Phase::Fixing {
8638 let restored = match self.state.winner().cloned() {
8639 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8640 None => Err(anyhow::anyhow!("repair has no winning branch")),
8641 };
8642 let why = match restored {
8643 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8644 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8645 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8646 };
8647 self.hold_merge_repair(i, why)?;
8648 return Ok(false);
8649 }
8650 let failure = if i >= self.state.config.graph.land_rounds
8651 || self.state.config.graph.review_rounds == 0
8652 {
8653 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8654 } else {
8655 None
8656 };
8657 if let Some(why) = failure {
8658 self.hold_merge_repair(i, why)?;
8659 return Ok(false);
8660 }
8661 if request.phase == Phase::Requested {
8662 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8663 Ok(pr) => pr,
8664 Err(e) => {
8665 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8666 return Ok(false);
8667 }
8668 };
8669 self.state.merge_repairs[i].phase = Phase::Fixing;
8670 self.state.land_approval = None;
8671 self.state.land_armed_head = None;
8672 self.state.save()?;
8673 let evidence = serde_json::to_string_pretty(&request)?;
8676 let reason = format!(
8677 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8678 );
8679 let budget = self.state.config.graph.land_rounds;
8680 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8681 match result {
8682 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8683 Ok(land::Fixed::Failed(why)) => {
8684 self.hold_merge_repair(
8685 i,
8686 format!(
8687 "repair failed: {why}; inspect the branch and run an explicit review"
8688 ),
8689 )?;
8690 return Ok(false);
8691 }
8692 Err(e) => {
8693 self.hold_merge_repair(
8694 i,
8695 format!(
8696 "repair failed: {e:#}; inspect the branch and run an explicit review"
8697 ),
8698 )?;
8699 return Ok(false);
8700 }
8701 }
8702 self.begin_merge_repair_review(i)?;
8706 }
8707 self.review_loop().await?;
8708 self.gate().await?;
8709 if self.resource_halt()? || self.state.parked {
8710 return Ok(false);
8711 }
8712 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8713 != Some(RunStatus::Gating)
8714 || !self.state.gate_allows_progress()
8715 {
8716 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8717 return Ok(false);
8718 }
8719 let winner = self
8722 .state
8723 .winner()
8724 .cloned()
8725 .context("repair has no winning branch")?;
8726 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8727 if self.state.gate_head.as_deref() != Some(after.as_str())
8728 || !git::is_clean(&winner.worktree).await?
8729 {
8730 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8731 return Ok(false);
8732 }
8733 let expected = self
8734 .state
8735 .pr_pushed_head
8736 .as_deref()
8737 .unwrap_or(&request.head)
8738 .to_owned();
8739 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8740 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8743 let refspec = format!("{after}:refs/heads/{public}");
8744 let push = git::git_raw(
8745 &winner.worktree,
8746 &["push", &lease, &self.state.config.merge.remote, &refspec],
8747 )
8748 .await;
8749 match push {
8750 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8751 Ok(out) => {
8752 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8753 return Ok(false);
8754 }
8755 Err(e) => {
8756 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8757 return Ok(false);
8758 }
8759 }
8760 self.state.merge_repairs[i].phase = Phase::Complete;
8761 self.state.status = RunStatus::Landing;
8762 self.state.event(
8763 "repair",
8764 "new review and gate completed; fresh owner merge decision required",
8765 );
8766 self.state.save()?;
8767 Ok(true)
8768 }
8769
8770 fn merge_repair_artifacts(&self) -> PathBuf {
8771 let artifacts = agent::artifacts_dir(&self.state.dir());
8772 match self
8773 .state
8774 .merge_repairs
8775 .last()
8776 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8777 {
8778 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8779 None => artifacts,
8780 }
8781 }
8782
8783 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8784 self.state.reviews.clear();
8785 self.state.gate.clear();
8786 self.state.gate_ran = false;
8787 self.state.gate_head = None;
8788 self.state.gate_fixes.clear();
8789 self.state.gate_repair_e2e = None;
8790 self.state.pre_gate.clear();
8791 self.state.pre_gate_commit = None;
8792 self.state.contested_handoff = None;
8793 self.state.land_reruns = None;
8794 self.state.land_approval = None;
8795 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8796 self.state.status = RunStatus::Reviewing;
8797 self.state.save()
8798 }
8799
8800 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8801 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8802 self.state.merge_repairs[index].failure = Some(why.clone());
8803 self.state.status = RunStatus::Blocked;
8804 self.state.event("repair", why);
8805 self.state.save()
8806 }
8807
8808 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8812 if let Some(existing) = self.state.seats.get(key)
8813 && existing.agent == agent
8814 {
8815 return existing.clone();
8816 }
8817 let fresh = if self.state.seats.contains_key(key) {
8820 handover_seat(key, agent, self.state.next_seat_seed())
8821 } else {
8822 SeatState::new(key, agent, self.state.seed)
8823 };
8824 self.state.seats.insert(key.to_owned(), fresh.clone());
8825 fresh
8826 }
8827
8828 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8834 match self.state.seats.get(key) {
8835 Some(s) if s.agent != spec.id => {
8836 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8837 }
8838 _ => spec,
8839 }
8840 }
8841
8842 fn view(&self, c: &Candidate) -> CandidateView {
8844 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8845 .unwrap_or_default();
8846 let (patch, _) = blind::sanitize_patch(
8847 &format!("candidate {} patch", c.label),
8848 &raw,
8849 &self.state.config.blind,
8850 );
8851 CandidateView {
8852 label: c.label,
8853 branch: c.branch.clone(),
8854 summary: c.summary.clone(),
8855 stat: c.stat.clone(),
8856 patch,
8857 }
8858 }
8859
8860 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8862 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8863 prompt::judge(
8864 "(see above)",
8865 &views,
8866 self.roles.judges.len(),
8867 base_short,
8868 "en",
8869 )
8870 }
8871
8872 fn vote_prompt_full(
8878 &self,
8879 j: usize,
8880 labels: &[char],
8881 language: &str,
8882 candidates: &[Candidate],
8883 base_short: &str,
8884 ) -> String {
8885 let mut text = format!(
8886 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8887 prompt::final_vote(labels, language),
8888 self.state.instruction,
8889 self.candidate_block(candidates, base_short)
8890 );
8891 if let Some(own) = self
8892 .state
8893 .judgements
8894 .get(j)
8895 .filter(|r| !r.ranking.is_empty())
8896 {
8897 let reasons = own
8898 .reasons
8899 .iter()
8900 .map(|(k, v)| format!("- {k}: {v}"))
8901 .collect::<Vec<_>>()
8902 .join("\n");
8903 text.push_str(&format!(
8904 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8905 own.ranking.iter().collect::<String>(),
8906 if reasons.is_empty() {
8907 ""
8908 } else {
8909 ", because:\n"
8910 }
8911 ));
8912 }
8913 if !self.state.deliberation.is_empty() {
8914 text.push_str("\n# What was argued before this vote\n");
8915 for t in self.transcript(&[], j) {
8916 text.push_str(&format!(
8917 "\n## {}{}\n\n{}\n",
8918 t.who,
8919 if t.is_self { " (you)" } else { "" },
8920 t.body.trim()
8921 ));
8922 }
8923 }
8924 text
8925 }
8926
8927 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8934 let mut turns = Vec::new();
8935 for j in &self.state.judgements {
8936 if j.ranking.is_empty() {
8937 continue;
8938 }
8939 let reasons = j
8940 .reasons
8941 .iter()
8942 .map(|(k, v)| format!("- {k}: {v}"))
8943 .collect::<Vec<_>>()
8944 .join("\n");
8945 turns.push(Turn {
8946 who: format!("Judge {} (opening ranking)", j.judge),
8947 is_self: j.judge == self_idx + 1,
8948 body: format!(
8949 "Ranked {}{}{reasons}",
8950 j.ranking.iter().collect::<String>(),
8951 if reasons.is_empty() {
8952 ""
8953 } else {
8954 ", because:\n"
8955 }
8956 ),
8957 });
8958 }
8959 for t in self
8960 .state
8961 .deliberation
8962 .iter()
8963 .flat_map(|r| r.turns.iter())
8964 .chain(current)
8965 {
8966 turns.push(Turn {
8967 who: format!("Judge {}", t.judge),
8968 is_self: t.judge == self_idx + 1,
8969 body: t.body.clone(),
8970 });
8971 }
8972 turns
8973 }
8974}
8975
8976fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8978 agent::has_session(spec.kind, seat, sessions)
8979}
8980
8981fn next_untried_in_roster<'a>(
9002 roster: &'a [AgentSpec],
9003 start: usize,
9004 tried: &BTreeSet<String>,
9005) -> Option<&'a AgentSpec> {
9006 roster
9007 .get(start + 1..)?
9008 .iter()
9009 .find(|s| !tried.contains(&s.id))
9010}
9011
9012fn pick_successor<'a>(
9033 roster: &'a [AgentSpec],
9034 start: usize,
9035 tried: &BTreeSet<String>,
9036 carried: Option<&BTreeSet<String>>,
9037 others: &BTreeSet<String>,
9038) -> Option<&'a AgentSpec> {
9039 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9040 roster
9041 .get(start + 1..)
9042 .and_then(|tail| {
9043 tail.iter()
9044 .filter(free)
9045 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9046 })
9047 .or_else(|| {
9048 carried
9049 .is_some()
9050 .then(|| roster.iter().find(free))
9051 .flatten()
9052 })
9053 .or_else(|| match carried {
9054 Some(c) => next_for_seat(roster, start, tried, c),
9055 None => next_untried_in_roster(roster, start, tried),
9056 })
9057}
9058
9059fn next_for_seat<'a>(
9069 roster: &'a [AgentSpec],
9070 start: usize,
9071 round_tried: &BTreeSet<String>,
9072 carried_failed: &BTreeSet<String>,
9073) -> Option<&'a AgentSpec> {
9074 roster
9075 .get(start + 1..)?
9076 .iter()
9077 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9078 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9079}
9080
9081fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9087 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9088 return spec;
9089 };
9090 let ok = |id: &str| !h.failed.contains(id);
9091 if let Some(last) = h.last_ok.as_deref()
9092 && ok(last)
9093 && let Some(s) = roster.iter().find(|s| s.id == last)
9094 {
9095 return s.clone();
9096 }
9097 if ok(&spec.id) {
9098 return spec;
9099 }
9100 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9101 roster
9102 .get(from + 1..)
9103 .into_iter()
9104 .flatten()
9105 .chain(roster.iter())
9106 .find(|s| ok(&s.id))
9107 .cloned()
9108 .unwrap_or(spec)
9109}
9110
9111pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9115 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9116}
9117
9118const TIMED_OUT: &str = "timed out";
9121
9122impl FailClass {
9123 fn of(out: &AgentOutcome) -> Option<Self> {
9125 match out {
9126 AgentOutcome::Ok(_) => None,
9127 AgentOutcome::Quota(_) => Some(Self::Quota),
9128 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9129 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9130 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9131 AgentOutcome::ResourceBlocked(_) => None,
9134 }
9135 }
9136
9137 fn stem_word(&self) -> &'static str {
9139 match self {
9140 Self::Quota => "quota",
9141 _ => "handover",
9142 }
9143 }
9144}
9145
9146fn failure_signature(msg: &str) -> String {
9150 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9151 if let Some(code) = line
9154 .strip_prefix("process exited with no usable output (exit code ")
9155 .and_then(|s| s.strip_suffix(')'))
9156 .and_then(|s| s.parse::<i32>().ok())
9157 {
9158 line = format!("exited with some({code}) and no usable output");
9159 } else if matches!(
9160 line.as_str(),
9161 "process exited with no usable output (no exit code reported)"
9162 | "process exited with no usable output (exit code unavailable)"
9163 ) {
9164 line = "exited with none and no usable output".to_owned();
9165 }
9166 let mut out = Vec::new();
9167 for word in line.split_whitespace() {
9168 if word.contains('/') || word.contains('\\') {
9169 out.push("<path>".to_owned());
9170 continue;
9171 }
9172 let mut w = String::new();
9173 let mut in_digits = false;
9174 for c in word.chars() {
9175 if c.is_ascii_digit() {
9176 if !in_digits {
9177 w.push('#');
9178 }
9179 in_digits = true;
9180 } else {
9181 in_digits = false;
9182 w.push(c);
9183 }
9184 }
9185 out.push(w);
9186 }
9187 out.join(" ").chars().take(120).collect()
9188}
9189
9190fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9197 match cur {
9198 FailClass::Quota | FailClass::Timeout => true,
9199 FailClass::Other(_) => prev != Some(cur),
9200 }
9201}
9202
9203fn fail_reason(out: &AgentOutcome) -> String {
9205 match out {
9206 AgentOutcome::Ok(_) => String::new(),
9207 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9208 AgentOutcome::Dropped(o) => format!(
9209 "the CLI dropped the stream ({})",
9210 o.dropped
9211 .as_ref()
9212 .map(|d| d.why.as_str())
9213 .unwrap_or("it ended without delivering its answer")
9214 ),
9215 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9216 e.lines().next().unwrap_or("").chars().take(160).collect()
9217 }
9218 }
9219}
9220
9221pub(crate) fn record_handover(
9224 state: &mut RunState,
9225 node: &str,
9226 seat: &str,
9227 from: &str,
9228 to: &str,
9229 class: &FailClass,
9230 reason: &str,
9231) {
9232 let message = if *class == FailClass::Quota {
9233 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9234 } else {
9235 format!("{seat}: handed over {from} -> {to} ({reason})")
9236 };
9237 state.event(node, message);
9238 state.handovers.push(Handover {
9239 at: Timestamp::now(),
9240 node: node.to_owned(),
9241 seat: seat.to_owned(),
9242 from: from.to_owned(),
9243 to: to.to_owned(),
9244 reason: reason.to_owned(),
9245 });
9246}
9247
9248fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9262 commands.iter().any(|c| c.exit_code.is_none())
9263}
9264
9265fn verified_noop_claim(
9278 usable: bool,
9279 commands: &[agent::CommandEvidence],
9280 text: &str,
9281) -> Option<String> {
9282 (usable && !has_unconfirmed_command(commands))
9283 .then(|| verdict::verified_noop(text))
9284 .flatten()
9285}
9286
9287fn short(commit: &str) -> String {
9288 commit.chars().take(7).collect()
9289}
9290
9291fn make_executable(path: &Path) -> Result<()> {
9292 #[cfg(unix)]
9293 {
9294 use std::os::unix::fs::PermissionsExt as _;
9295 let mut perms = std::fs::metadata(path)?.permissions();
9296 perms.set_mode(0o755);
9297 std::fs::set_permissions(path, perms)?;
9298 }
9299 #[cfg(not(unix))]
9300 {
9301 let _ = path;
9302 }
9303 Ok(())
9304}
9305
9306struct WaveCtx<'a> {
9313 run: &'a str,
9316 node: &'a str,
9318 prompts: &'a Prompts,
9319 cache: Option<&'a Path>,
9321 round: Option<usize>,
9324 carry_seats: bool,
9328 pauses: Option<(&'a Pause, &'a Pause)>,
9331}
9332
9333fn repository_wait(
9335 state: &RunState,
9336 pauses: Option<(&Pause, &Pause)>,
9337) -> crate::repo_lock::WaitPolicy {
9338 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9339 let inherited = crate::repo_lock::WaitPolicy::current();
9340 crate::repo_lock::WaitPolicy::new(
9341 Duration::from_secs(state.config.graph.timeout_resource_wait),
9342 move || {
9343 inherited.cancelled()
9344 || pauses
9345 .as_ref()
9346 .is_some_and(|(a, b)| a.parked() || b.parked())
9347 },
9348 )
9349}
9350
9351fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9352 if error.is::<crate::repo_lock::ResourceBlocked>() {
9353 state.resource_block = Some(crate::run::ResourceBlock {
9354 cause: crate::run::ResourceBlockCause::Other,
9355 node: "repository".into(),
9356 kind: if error.to_string().contains("disk pressure") {
9357 "disk pressure"
9358 } else {
9359 "repository coordination"
9360 }
9361 .into(),
9362 detail: error.to_string(),
9363 at: Timestamp::now(),
9364 });
9365 state.event("resource", error.to_string());
9366 }
9367}
9368
9369async fn coordinate_operation(
9370 state: &mut RunState,
9371 cwd: &Path,
9372 wait: &crate::repo_lock::WaitPolicy,
9373) -> Result<Option<crate::repo_lock::SeatLease>> {
9374 let result =
9375 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9376 .await;
9377 if let Err(e) = &result {
9378 record_repository_block(state, e);
9379 }
9380 result
9381}
9382
9383async fn run_one(
9385 job: SeatJob,
9386 sem: Arc<Semaphore>,
9387 ctx: &WaveCtx<'_>,
9388 state: &mut RunState,
9389 attempt: usize,
9390) -> (SeatState, AgentOutcome) {
9391 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9392 .await
9393 .pop()
9394 .expect("one job in, one result out");
9395 (seat, out)
9396}
9397
9398async fn wave(
9404 jobs: Vec<SeatJob>,
9405 sem: Arc<Semaphore>,
9406 ctx: &WaveCtx<'_>,
9407 state: &mut RunState,
9408 attempt: usize,
9409) -> Vec<(usize, SeatState, AgentOutcome)> {
9410 let WaveCtx {
9411 run,
9412 node,
9413 prompts,
9414 cache,
9415 round,
9416 carry_seats: _,
9417 pauses,
9418 } = *ctx;
9419 for job in &jobs {
9420 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9421 }
9422 if let Err(e) = state.save() {
9423 tracing::warn!("could not persist in-progress seats: {e:#}");
9428 }
9429 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9445 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9449 let cache_guard = if let Some(cache_dir) = cache {
9450 if jobs_had_a_writer {
9451 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9452 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9453 match acquire_cache_lease(
9454 state,
9455 cache_dir,
9456 &owner,
9457 resource_limit,
9458 node,
9459 node,
9460 &cancel,
9461 )
9462 .await
9463 {
9464 Ok(Some(g)) => Some(g),
9465 Ok(None) => {
9468 if let Some(block) = &mut state.resource_block {
9469 block.cause = crate::run::ResourceBlockCause::Cancelled {
9470 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9471 };
9472 }
9473 return block_wave(
9474 jobs,
9475 state,
9476 node,
9477 attempt,
9478 "parked while waiting for the shared build cache",
9479 );
9480 }
9481 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9484 Err(e) => {
9485 let why = format!("{e:#}");
9489 state.event(
9490 "resource",
9491 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9492 );
9493 return block_wave(jobs, state, node, attempt, &why);
9494 }
9495 }
9496 } else {
9497 None
9498 }
9499 } else {
9500 None
9501 };
9502 state.seats_spawning(node, attempt);
9505 if let Err(e) = state.save() {
9506 tracing::warn!("could not persist the start of a wave: {e:#}");
9507 }
9508 let slot_limit = resource_limit
9512 + jobs
9513 .iter()
9514 .map(|j| j.timeout)
9515 .max()
9516 .unwrap_or_default()
9517 .saturating_mul(jobs.len() as u32);
9518 let mut set = tokio::task::JoinSet::new();
9519 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9522 let overlay = prompts.overlay(node);
9523 let repository_wait = repository_wait(state, pauses);
9524 for (i, mut job) in jobs.into_iter().enumerate() {
9525 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9526 if cache.is_some() {
9527 job.prompt.push('\n');
9528 job.prompt
9529 .push_str(&prompt::build_cache_note(node, job.allow_write));
9530 }
9531 let sem = Arc::clone(&sem);
9532 let run = run.to_owned();
9533 let node = node.to_owned();
9534 let attachments = if node == "implement" {
9537 state.attachments.clone()
9538 } else {
9539 Vec::new()
9540 };
9541 let cache = cache
9552 .filter(|_| job.allow_write && cache_guard.is_some())
9553 .map(Path::to_path_buf);
9554 let spawned_tx = spawned_tx.clone();
9555 let repository_wait = repository_wait.clone();
9556 let coordination = state.config.coordination.clone();
9557 let editing_network = state.config.graph.editing_network;
9558 let fetched_base = state.fetched_base.clone();
9559 set.spawn(async move {
9560 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9564 Ok(p) => p,
9565 Err(_) => {
9566 return (
9567 i,
9568 job.seat,
9569 AgentOutcome::ResourceBlocked(format!(
9570 "no parallel slot became free within {}s",
9571 slot_limit.as_secs()
9572 )),
9573 false,
9574 );
9575 }
9576 };
9577 spawned_tx.send(job.seat.key.clone()).ok();
9578 let mut seat = job.seat;
9579 let out = repository_wait
9580 .scope(agent::invoke(
9581 &job.spec,
9582 &mut seat,
9583 &Invocation {
9584 coordination: coordination.as_ref(),
9585 editing_network,
9586 fetched_base: fetched_base.as_ref(),
9587 cwd: &job.cwd,
9588 prompt: &job.prompt,
9589 timeout: job.timeout,
9590 allow_write: job.allow_write,
9591 unsandboxed: false,
9592 sessions: job.sessions,
9593 artifacts: &job.artifacts,
9594 stem: &job.stem,
9595 run: &run,
9596 node: &node,
9597 cache_dir: cache.as_deref(),
9598 attachments: &attachments,
9599 writable: &[],
9600 },
9601 ))
9602 .await;
9603 let interrupted = out
9604 .as_ref()
9605 .err()
9606 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9607 let out = match out {
9608 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9609 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9610 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9618 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9619 Ok(o) => AgentOutcome::Failed(format!(
9620 "process exited with no usable output ({})",
9621 crate::notices::exit_status(o.exit_code)
9622 )),
9623 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9624 AgentOutcome::ResourceBlocked(e.to_string())
9625 }
9626 Err(e) => AgentOutcome::Failed(e.to_string()),
9627 };
9628 (i, seat, out, interrupted)
9629 });
9630 }
9631 drop(spawned_tx);
9632 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9633 loop {
9634 let joined = tokio::select! {
9635 Some(key) = spawned_rx.recv() => {
9636 state.seat_spawned(&key);
9638 if let Err(e) = state.save() {
9639 tracing::warn!("could not persist a seat's start: {e:#}");
9640 }
9641 continue;
9642 }
9643 joined = set.join_next() => joined,
9644 };
9645 let Some(joined) = joined else {
9646 break;
9647 };
9648 let (i, seat, out, interrupted) = match joined {
9649 Ok(v) => v,
9650 Err(e) => {
9654 tracing::error!("agent task panicked: {e}");
9655 continue;
9656 }
9657 };
9658 state.seat_finished(&seat.key);
9659 if interrupted
9660 && node == "implement"
9661 && let Some(label) = state
9662 .candidates
9663 .iter()
9664 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9665 .map(|candidate| candidate.label)
9666 {
9667 state.candidate_preservation.insert(
9668 label,
9669 "interrupted implementation awaiting persistence".into(),
9670 );
9671 state.interrupted_implementations.insert(label);
9672 }
9673 if let AgentOutcome::ResourceBlocked(why) = &out
9674 && state.resource_block.is_none()
9675 {
9676 state.resource_block = Some(crate::run::ResourceBlock {
9677 cause: crate::run::ResourceBlockCause::Other,
9678 node: node.to_owned(),
9679 kind: if why.contains("disk pressure") {
9680 "disk pressure"
9681 } else if why.contains("repository coordination") {
9682 "repository coordination"
9683 } else {
9684 "parallel slot"
9685 }
9686 .to_owned(),
9687 detail: why.clone(),
9688 at: Timestamp::now(),
9689 });
9690 let activity = if interrupted {
9691 "agent execution interrupted"
9692 } else {
9693 "no agent was started"
9694 };
9695 state.event("resource", format!("{node}: {activity} - {why}"));
9696 }
9697 record_jobs(state, node, round, &seat.key, &out);
9698 if let Err(e) = state.save() {
9699 tracing::warn!("could not persist a seat's completion: {e:#}");
9700 }
9701 if collected.len() <= i {
9702 collected.resize_with(i + 1, || None);
9703 }
9704 collected[i] = Some((i, seat, out));
9705 }
9706 if state
9712 .active
9713 .values()
9714 .any(|a| a.node == node && a.attempt == attempt)
9715 {
9716 state
9717 .active
9718 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9719 if let Err(e) = state.save() {
9720 tracing::warn!("could not persist the end of a wave: {e:#}");
9721 }
9722 }
9723 if let Some(cache_dir) = cache
9730 && jobs_had_a_writer
9731 {
9732 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9733 }
9734 if let Some(guard) = cache_guard {
9735 guard.release();
9736 }
9737 collected.into_iter().flatten().collect()
9738}
9739
9740fn record_jobs(
9751 state: &mut RunState,
9752 node: &str,
9753 round: Option<usize>,
9754 seat: &str,
9755 out: &AgentOutcome,
9756) {
9757 let commands: &[agent::CommandEvidence] = match out {
9758 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9759 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9760 };
9761 let checked_at = Timestamp::now();
9762 for c in commands {
9763 state.jobs.push(JobRecord {
9764 node: node.to_owned(),
9765 round,
9766 seat: seat.to_owned(),
9767 id: c.id.clone(),
9768 description: c.description.clone(),
9769 checked_at,
9770 status: match c.exit_code {
9771 Some(0) => JobStatus::Completed,
9772 Some(_) => JobStatus::Failed,
9773 None => JobStatus::Unknown,
9774 },
9775 exit_code: c.exit_code,
9776 result_summary: c.result_summary.clone(),
9777 source: c.source.clone(),
9778 });
9779 }
9780}
9781
9782fn round_is_clean(
9803 blocking: usize,
9804 e2e_ok: bool,
9805 answered: usize,
9806 expected: usize,
9807 quota_missing: usize,
9808 policy: IncompleteReviewPolicy,
9809) -> bool {
9810 if blocking != 0 || !e2e_ok {
9811 return false;
9812 }
9813 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9814 return true;
9815 }
9816 answered > 0 && expected - answered <= quota_missing
9817}
9818
9819pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9843 if max_rounds == 0
9844 || reviews.last().is_some_and(|r| {
9845 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9846 })
9847 {
9848 return Some(RunStatus::Gating);
9849 }
9850 let last = reviews.last()?;
9851 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9852 if reviews.len() < max_rounds && !stagnant {
9853 return None;
9854 }
9855 if last.incomplete() && last.blocking == 0 {
9856 return Some(RunStatus::Blocked);
9857 }
9858 if last.e2e_status() == E2eStatus::ResourceBlocked
9859 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9860 {
9861 return None;
9862 }
9863 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9864 RunStatus::Gating
9865 } else {
9866 RunStatus::Blocked
9867 })
9868}
9869
9870pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9885 if nudged {
9886 (full / 4).max(Duration::from_secs(120)).min(full)
9887 } else {
9888 full
9889 }
9890}
9891
9892#[allow(clippy::too_many_arguments)]
9921async fn ask_json_wave<T>(
9922 jobs: Vec<SeatJob>,
9923 sem: Arc<Semaphore>,
9924 retries: usize,
9925 roster: &[AgentSpec],
9926 ctx: &WaveCtx<'_>,
9927 losses: &mut Vec<QuotaLoss>,
9928 state: &mut RunState,
9929 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9930) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9931where
9932 T: serde::de::DeserializeOwned + Send + 'static,
9933{
9934 ask_wave_with(
9935 jobs,
9936 sem,
9937 retries,
9938 roster,
9939 ctx,
9940 losses,
9941 state,
9942 &|text: &str| {
9943 let v = verdict::extract_json::<T>(text)?;
9944 validate(&v)?;
9945 Ok(v)
9946 },
9947 )
9948 .await
9949}
9950
9951#[allow(clippy::too_many_arguments)]
9960async fn ask_wave_with<T>(
9961 jobs: Vec<SeatJob>,
9962 sem: Arc<Semaphore>,
9963 retries: usize,
9964 roster: &[AgentSpec],
9965 ctx: &WaveCtx<'_>,
9966 losses: &mut Vec<QuotaLoss>,
9967 state: &mut RunState,
9968 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9969) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9970where
9971 T: Send + 'static,
9972{
9973 let n = jobs.len();
9974 let originals: Vec<SeatJob> = jobs;
9975 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9976 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9977 let mut nudges: Vec<usize> = vec![0; n];
9984 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9988 let mut tried: Vec<BTreeSet<String>> = specs
9989 .iter()
9990 .map(|s| BTreeSet::from([s.id.clone()]))
9991 .collect();
9992 let starts: Vec<usize> = specs
9993 .iter()
9994 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9995 .collect();
9996 let carry = ctx.carry_seats && !roster.is_empty();
9997 let carried: Vec<BTreeSet<String>> = originals
10000 .iter()
10001 .map(|j| {
10002 state
10003 .seat_history
10004 .get(&j.seat.key)
10005 .filter(|_| carry)
10006 .map(|h| h.failed.clone())
10007 .unwrap_or_default()
10008 })
10009 .collect();
10010 let mut prev: Vec<Option<FailClass>> = originals
10011 .iter()
10012 .map(|j| {
10013 state
10014 .seat_history
10015 .get(&j.seat.key)
10016 .filter(|_| carry)
10017 .and_then(|h| h.last_fail.clone())
10018 })
10019 .collect();
10020 let next_agent =
10021 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10022 let others: BTreeSet<String> = specs
10023 .iter()
10024 .enumerate()
10025 .filter(|(j, _)| *j != i)
10026 .map(|(_, s)| s.id.clone())
10027 .collect();
10028 pick_successor(
10029 roster,
10030 starts[i],
10031 tried,
10032 carry.then(|| &carried[i]),
10033 &others,
10034 )
10035 .cloned()
10036 };
10037 let mut fresh: Vec<Option<String>> = vec![None; n];
10038 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10039 let mut pending: Vec<usize> = (0..n).collect();
10040
10041 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10046 for round in 0..max_rounds {
10047 if pending.is_empty() {
10048 break;
10049 }
10050 let mut batch = Vec::with_capacity(pending.len());
10051 let mut renudged: Vec<&str> = Vec::new();
10052 for &i in &pending {
10053 let src = &originals[i];
10054 let full: &str = match &src.handover {
10059 Some(h) if specs[i].id != src.spec.id => h,
10060 _ => &src.prompt,
10061 };
10062 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10065 (
10066 full.to_owned(),
10067 src.timeout,
10068 format!("{}-{word}-{}", src.stem, specs[i].id),
10069 )
10070 } else if nudges[i] == 0 {
10071 (full.to_owned(), src.timeout, src.stem.clone())
10072 } else {
10073 renudged.push(src.seat.key.as_str());
10074 let why = done[i]
10075 .as_ref()
10076 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10077 .unwrap_or_else(|| "no parsable answer".to_owned());
10078 let nudge = prompt::nudge(&why);
10079 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10080 let prompt = if nudged {
10081 nudge
10082 } else {
10083 format!("{full}\n\n---\n\n{nudge}")
10084 };
10085 (
10086 prompt,
10087 retry_budget(src.timeout, nudged),
10088 format!("{}-retry{}", src.stem, nudges[i]),
10089 )
10090 };
10091 batch.push(SeatJob {
10092 spec: specs[i].clone(),
10093 seat: seats[i].clone(),
10094 cwd: src.cwd.clone(),
10095 prompt,
10096 timeout,
10097 allow_write: src.allow_write,
10098 sessions: src.sessions,
10099 artifacts: src.artifacts.clone(),
10100 stem,
10101 handover: None,
10102 });
10103 }
10104
10105 if !renudged.is_empty() {
10106 state.event(
10107 ctx.node,
10108 format!("retry {round}: re-asking {}", renudged.join(", ")),
10109 );
10110 }
10111 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10112 let mut still = Vec::new();
10113 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10114 seats[i] = seat;
10115 if let AgentOutcome::ResourceBlocked(why) = &out {
10118 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10119 continue;
10120 }
10121 let class = FailClass::of(&out);
10122 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10125 && nudges[i] < retries
10126 && !roster.is_empty();
10127 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10128 let next = should_hand_over(prev[i].as_ref(), &cur)
10129 .then(|| next_agent(i, &tried[i], &specs))
10130 .flatten();
10131 if carry {
10132 let h = state
10133 .seat_history
10134 .entry(originals[i].seat.key.clone())
10135 .or_default();
10136 h.failed.insert(specs[i].id.clone());
10137 h.last_fail = Some(cur.clone());
10138 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10139 h.last_ok = None;
10140 }
10141 if let Err(e) = state.save() {
10144 tracing::warn!("could not persist a seat's failure history: {e:#}");
10145 }
10146 }
10147 if let Some(next) = next {
10148 record_handover(
10149 state,
10150 ctx.node,
10151 &originals[i].seat.key,
10152 &specs[i].id,
10153 &next.id,
10154 &cur,
10155 &fail_reason(&out),
10156 );
10157 tried[i].insert(next.id.clone());
10158 prev[i] = Some(cur.clone());
10159 seats[i] =
10160 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10161 specs[i] = next;
10162 fresh[i] = Some(cur.stem_word().to_owned());
10163 nudges[i] = 0;
10164 done[i] = Some(Err(anyhow::anyhow!(
10165 "handed over after: {}",
10166 fail_reason(&out)
10167 )));
10168 still.push(i);
10169 continue;
10170 }
10171 }
10172 if carry
10173 && !nudge_first
10174 && let Some(cur) = class.clone()
10175 {
10176 let h = state
10178 .seat_history
10179 .entry(originals[i].seat.key.clone())
10180 .or_default();
10181 h.failed.insert(specs[i].id.clone());
10182 h.last_fail = Some(cur);
10183 }
10184 let parsed = match out {
10185 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10186 AgentOutcome::Quota(o) => {
10187 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10188 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10189 }
10190 AgentOutcome::Dropped(o) => {
10194 let why = o
10195 .dropped
10196 .as_ref()
10197 .map(|d| d.why.as_str())
10198 .unwrap_or("the CLI ended the stream without delivering its answer");
10199 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10200 }
10201 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10202 Err(anyhow::anyhow!(e))
10203 }
10204 };
10205 let quota = class == Some(FailClass::Quota);
10206 let failed = parsed.is_err();
10207 done[i] = Some(parsed);
10208 if carry && !failed {
10209 let h = state
10210 .seat_history
10211 .entry(originals[i].seat.key.clone())
10212 .or_default();
10213 h.failed.remove(&specs[i].id);
10214 h.last_ok = Some(specs[i].id.clone());
10215 h.last_fail = None;
10216 }
10217 let agent_failure = class.is_some()
10225 && !nudge_first
10226 && !roster.is_empty()
10227 && next_agent(i, &tried[i], &specs).is_some();
10228 if failed && !quota && !agent_failure && nudges[i] < retries {
10229 nudges[i] += 1;
10230 still.push(i);
10231 }
10232 }
10233 pending = still;
10234 }
10235
10236 for (i, q) in last_quota.into_iter().enumerate() {
10239 if let Some(reset) = q {
10240 losses.push(QuotaLoss {
10241 seat: originals[i].seat.key.clone(),
10242 node: ctx.node.to_owned(),
10243 at: Timestamp::now(),
10244 reset,
10245 });
10246 }
10247 }
10248
10249 seats
10250 .into_iter()
10251 .zip(done)
10252 .zip(nudges)
10253 .map(|((seat, res), attempts)| {
10254 (
10255 seat,
10256 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10257 attempts,
10258 )
10259 })
10260 .collect()
10261}
10262
10263fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10265 results
10266 .iter()
10267 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10268}
10269
10270fn block_wave(
10274 jobs: Vec<SeatJob>,
10275 state: &mut RunState,
10276 node: &str,
10277 attempt: usize,
10278 why: &str,
10279) -> Vec<(usize, SeatState, AgentOutcome)> {
10280 state.resource_wait = None;
10281 state
10282 .active
10283 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10284 if let Err(e) = state.save() {
10285 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10286 }
10287 jobs.into_iter()
10288 .enumerate()
10289 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10290 .collect()
10291}
10292
10293async fn acquire_cache_lease(
10306 state: &mut RunState,
10307 cache_dir: &Path,
10308 owner: &crate::cache::Owner,
10309 budget: Duration,
10310 node: &str,
10311 context: &str,
10312 cancel: &(dyn Fn() -> bool + Sync),
10313) -> Result<Option<crate::cache::Guard>> {
10314 let home = crate::run::home();
10315 let started = Instant::now();
10316 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10317 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10318 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10319 Err(e) => {
10320 state.event(
10321 "verify",
10322 format!("{context}: could not check the shared build cache: {e:#}"),
10323 );
10324 if let Err(e2) = state.save() {
10325 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10326 }
10327 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10328 }
10329 };
10330 state.event(
10331 "verify",
10332 format!(
10333 "{context}: waiting for the shared build cache at {} ({})",
10334 cache_dir.display(),
10335 busy.describe()
10336 ),
10337 );
10338 state.resource_wait = Some(crate::run::ResourceWait {
10341 node: context.to_owned(),
10342 kind: "shared build cache".to_owned(),
10343 holder: busy.describe(),
10344 since: Timestamp::now(),
10345 limit_secs: budget.as_secs(),
10346 });
10347 if let Err(e) = state.save() {
10348 tracing::warn!("could not persist a cache wait: {e:#}");
10349 }
10350 let remaining = budget.saturating_sub(started.elapsed());
10351 let waited = crate::cache::wait_for_unless(
10352 &home,
10353 cache_dir,
10354 owner,
10355 remaining,
10356 Duration::from_secs(5),
10357 cancel,
10358 )
10359 .await;
10360 state.resource_wait = None;
10361 let cause = match &waited {
10362 Ok(Some(_)) => None,
10363 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10364 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10365 Some(crate::run::ResourceBlockCause::Timeout {
10366 waited_secs: started.elapsed().as_secs(),
10367 limit_secs: budget.as_secs(),
10368 })
10369 }
10370 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10371 };
10372 if let Some(cause) = cause {
10373 let detail = match &waited {
10374 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10375 Err(error) => format!("{error:#}"),
10376 };
10377 state.resource_block = Some(crate::run::ResourceBlock {
10378 cause,
10379 node: node.to_owned(),
10380 kind: "shared build cache".into(),
10381 detail,
10382 at: Timestamp::now(),
10383 });
10384 state.event(
10385 "resource",
10386 state.resource_block.as_ref().unwrap().description(),
10387 );
10388 }
10389 if let Err(error) = state.save() {
10390 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10391 }
10392 waited
10393}
10394
10395#[derive(Debug)]
10400struct CacheCheckError(String);
10401
10402impl std::fmt::Display for CacheCheckError {
10403 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10404 f.write_str(&self.0)
10405 }
10406}
10407
10408impl std::error::Error for CacheCheckError {}
10409
10410fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10412 state
10413 .resource_block
10414 .get_or_insert_with(|| crate::run::ResourceBlock {
10415 cause: crate::run::ResourceBlockCause::Other,
10416 node: node.to_owned(),
10417 kind: "shared build cache".into(),
10418 detail: format!("{error:#}"),
10419 at: Timestamp::now(),
10420 });
10421 if let Err(error) = state.save() {
10422 tracing::warn!("could not persist cache resource block: {error:#}");
10423 }
10424}
10425
10426#[allow(clippy::too_many_arguments)]
10447async fn with_cache_lease<'s, F, Fut>(
10448 state: &'s mut RunState,
10449 cache_dir: Option<&Path>,
10450 node: &str,
10451 seat: &str,
10452 worktree: &Path,
10453 head: &str,
10454 budget: Duration,
10455 context: &str,
10456 wait: &crate::repo_lock::WaitPolicy,
10457 body: F,
10458) -> Result<(Vec<CommandOutcome>, bool)>
10459where
10460 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10461 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10462{
10463 let started = Instant::now();
10466 let home = crate::run::home();
10467 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10468 let guard = if let Some(cache_dir) = cache_dir {
10469 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10470 match acquire_cache_lease(
10471 state,
10472 cache_dir,
10473 &owner,
10474 resource_limit,
10475 node,
10476 context,
10477 &|| wait.cancelled(),
10478 )
10479 .await
10480 .and_then(|g| {
10481 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10482 }) {
10483 Ok(g) => Some(g),
10484 Err(e) => {
10485 record_cache_error(state, node, &e);
10486 return Ok((
10487 vec![CommandOutcome {
10488 command: "(waiting for the shared build cache)".into(),
10489 code: None,
10490 output_tail: e.to_string(),
10491 duration_ms: started.elapsed().as_millis() as u64,
10492 resource_blocked: true,
10493 }],
10494 false,
10495 ));
10496 }
10497 }
10498 } else {
10499 None
10500 };
10501 let _coordination = coordinate_operation(state, worktree, wait).await?;
10502 if let Some(cache_dir) = cache_dir {
10503 let identity = crate::cache::Identity::new(worktree, head);
10504 if let Err(e) = crate::cache::ensure_fresh(
10505 &home,
10506 cache_dir,
10507 &identity,
10508 &state.config.verify.cargo_manifests(),
10509 ) {
10510 record_cache_error(state, node, &e);
10511 state.event(
10512 "verify",
10513 format!(
10514 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10515 worktree.display(),
10516 short(head)
10517 ),
10518 );
10519 if let Some(guard) = guard {
10520 guard.release();
10521 }
10522 return Ok((
10523 vec![CommandOutcome {
10524 command: "(confirming the shared build cache is fresh)".into(),
10525 code: None,
10526 output_tail: e.to_string(),
10527 duration_ms: started.elapsed().as_millis() as u64,
10528 resource_blocked: true,
10529 }],
10530 false,
10531 ));
10532 }
10533 }
10534 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10535 if !timed_out_pids.is_empty() {
10536 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10537 }
10538 drop(_coordination);
10541 if let Some(guard) = guard {
10542 guard.release();
10543 }
10544 Ok((outcomes, retried))
10545}
10546
10547async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10559 wait_for_pids_with(
10560 pids,
10561 crate::proc::pid_alive,
10562 LEASE_RELEASE_POLL,
10563 LEASE_RELEASE_MAX_WAIT,
10564 )
10565 .await;
10566}
10567
10568async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10574 pids: &[u32],
10575 alive: F,
10576 poll: Duration,
10577 max_wait: Duration,
10578) {
10579 let deadline = Instant::now() + max_wait;
10580 loop {
10581 if pids.iter().all(|&pid| !alive(pid)) {
10582 return;
10583 }
10584 if Instant::now() >= deadline {
10585 return;
10586 }
10587 tokio::time::sleep(poll).await;
10588 }
10589}
10590
10591fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10598 outcomes.iter().any(|o| o.resource_blocked)
10599}
10600
10601pub(crate) enum GateFix {
10603 Retry,
10605 Stop,
10608 Defer,
10611}
10612
10613fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10621 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10622 red.peek().is_some()
10623 && red.all(|o| {
10624 !o.resource_blocked
10625 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10626 && !o.output_tail.trim().is_empty()
10627 })
10628}
10629
10630fn e2e_outcome_label(o: &CommandOutcome) -> String {
10634 if o.ok() {
10635 return "pass".to_owned();
10636 }
10637 let reason = if o.build_failed() {
10638 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10639 } else {
10640 format!("FAIL ({:?})", o.code)
10641 };
10642 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10643}
10644
10645async fn run_e2e_with_retry(
10653 state: &mut RunState,
10654 shell: &[String],
10655 commands: &[String],
10656 worktree: &Path,
10657 timeout: Duration,
10658 context: &str,
10659) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10660 let (mut e2e, mut timed_out_pids) = run_commands(
10661 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10662 )
10663 .await;
10664 for o in &e2e {
10665 state.event(
10666 "verify",
10667 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10668 );
10669 }
10670 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10674 if verify_retried {
10675 state.event(
10676 "verify",
10677 format!(
10678 "{context}: verify could not build/link, not a test result — retrying once \
10679 before concluding"
10680 ),
10681 );
10682 let retried = run_commands(
10683 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10684 )
10685 .await;
10686 e2e = retried.0;
10687 timed_out_pids.extend(retried.1);
10690 for o in &e2e {
10691 state.event(
10692 "verify",
10693 format!(
10694 "{context}: retry `{}` -> {}",
10695 o.command,
10696 e2e_outcome_label(o)
10697 ),
10698 );
10699 }
10700 }
10701 (e2e, verify_retried, timed_out_pids)
10702}
10703
10704#[allow(clippy::too_many_arguments)]
10720async fn run_commands(
10721 state: &mut RunState,
10722 node: &str,
10723 task: &str,
10724 attempt: usize,
10725 shell: &[String],
10726 commands: &[String],
10727 cwd: &Path,
10728 timeout: Duration,
10729) -> (Vec<CommandOutcome>, Vec<u32>) {
10730 if commands.is_empty() {
10731 return (Vec::new(), Vec::new());
10736 }
10737 let mut out = Vec::new();
10738 let mut timed_out_pids = Vec::new();
10739 let total = commands.len();
10740 for (idx, command) in commands.iter().enumerate() {
10741 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10742 if let Err(e) = state.save() {
10743 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10744 }
10745 let started = Instant::now();
10746 let mut cmd = tokio::process::Command::new(&shell[0]);
10747 cmd.quiet();
10748 for key in crate::worker::INVOCATION_ENV {
10752 cmd.env_remove(key);
10753 }
10754 cmd.args(&shell[1..])
10755 .arg(command)
10756 .current_dir(cwd)
10757 .stdin(std::process::Stdio::null())
10758 .stdout(std::process::Stdio::piped())
10759 .stderr(std::process::Stdio::piped())
10760 .kill_on_drop(true);
10761 let mut outputs = Vec::new();
10762 let protected = (|| -> Result<()> {
10763 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10764 outputs.push(guard);
10765 }
10766 if let Some(cache) = state.config.cache_dir() {
10767 outputs.push(crate::retention::OutputLease::cache(
10768 &crate::run::home(),
10769 &cache,
10770 false,
10771 )?);
10772 }
10773 Ok(())
10774 })();
10775 if let Err(e) = protected {
10776 out.push(CommandOutcome {
10777 command: command.clone(),
10778 code: None,
10779 output_tail: format!("output ownership unavailable: {e:#}"),
10780 duration_ms: 0,
10781 resource_blocked: true,
10782 });
10783 continue;
10784 }
10785 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10786 let (code, body) = match spawned {
10787 Ok(mut running) => {
10788 let pid = running.child().id();
10789 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10790 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10791 let mut paths = vec![cwd.to_owned()];
10792 if let Some(cache) = state.config.cache_dir() {
10793 paths.push(cache);
10794 }
10795 let mut pressure = None;
10796 let status = tokio::select! {
10797 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10798 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10799 pressure = Some(reason); None
10800 }
10801 };
10802 let (code, failure) = match status {
10803 Some(Ok(Ok(status))) => (
10804 status.code(),
10805 status
10806 .code()
10807 .is_none()
10808 .then(|| format!("command terminated: {status}")),
10809 ),
10810 Some(Ok(Err(error))) => {
10811 (None, Some(format!("failed waiting for command: {error}")))
10812 }
10813 Some(Err(_)) => {
10814 if !running.coordinated()
10818 && let Some(pid) = pid
10819 {
10820 timed_out_pids.push(pid);
10821 }
10822 (
10823 None,
10824 Some(format!("timed out after {}s", timeout.as_secs())),
10825 )
10826 }
10827 None => (None, None),
10828 };
10829 running.finish().await;
10830 if let Some(reason) = pressure {
10831 state.resource_block = Some(crate::run::ResourceBlock {
10832 cause: crate::run::ResourceBlockCause::Other,
10833 node: node.to_owned(),
10834 kind: "disk pressure".to_owned(),
10835 detail: reason.clone(),
10836 at: Timestamp::now(),
10837 });
10838 out.push(CommandOutcome {
10839 command: command.clone(),
10840 code: None,
10841 output_tail: reason,
10842 duration_ms: started.elapsed().as_millis() as u64,
10843 resource_blocked: true,
10844 });
10845 break;
10846 }
10847 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10848 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10849 if let Some(failure) = failure {
10850 body.push('\n');
10851 body.push_str(&failure);
10852 }
10853 (code, body)
10854 }
10855 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10856 };
10857 out.push(CommandOutcome {
10858 command: command.clone(),
10859 code,
10860 output_tail: tail(&body, OUTPUT_TAIL),
10861 duration_ms: started.elapsed().as_millis() as u64,
10862 resource_blocked: false,
10863 });
10864 }
10865 state.task_finished(task);
10866 if let Err(e) = state.save() {
10867 tracing::warn!("could not persist the end of {task}: {e:#}");
10868 }
10869 (out, timed_out_pids)
10870}
10871
10872fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10884 let repo = repo.display();
10885 match style {
10886 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10887 MergeStyle::Squash => {
10888 let subject = message
10891 .lines()
10892 .next()
10893 .unwrap_or(branch)
10894 .replace(['\\', '"', '$', '`'], "");
10895 format!(
10896 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10897 )
10898 }
10899 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10900 }
10901}
10902
10903const PR_TITLE_MAX: usize = 240;
10914
10915fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10926 let line = queue::first_line(text)?;
10927 let chars: Vec<char> = line.chars().collect();
10928 let (kept, mark) = if chars.len() <= max {
10929 (line.to_owned(), "")
10930 } else if let Some(end) = sentence_end(&chars, max) {
10931 (chars[..end].iter().collect::<String>(), "")
10932 } else {
10933 let room = max.saturating_sub(3);
10934 let cut = if chars[room].is_whitespace() {
10937 room
10938 } else {
10939 chars[..room]
10940 .iter()
10941 .rposition(|c| c.is_whitespace())
10942 .unwrap_or(room)
10943 };
10944 let head: String = chars[..cut].iter().collect();
10945 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10946 (head.to_owned(), "...")
10947 };
10948 if kept.is_empty() {
10949 return None;
10950 }
10951 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10952 return None;
10953 }
10954 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10955 return None;
10956 }
10957 Some(format!("{kept}{mark}"))
10958}
10959
10960fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10964 const MIN: usize = 20;
10965 for i in MIN..max.min(chars.len()) {
10966 if !matches!(chars[i], '.' | '!' | '?') {
10967 continue;
10968 }
10969 let Some(&next) = chars.get(i + 1) else {
10970 continue;
10971 };
10972 if !next.is_whitespace() {
10973 continue;
10974 }
10975 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10976 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10977 continue;
10978 }
10979 let word: String = chars[..i]
10980 .iter()
10981 .rev()
10982 .take_while(|c| !c.is_whitespace())
10983 .collect::<Vec<_>>()
10984 .into_iter()
10985 .rev()
10986 .collect();
10987 let word = word.to_ascii_lowercase();
10988 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10989 continue;
10990 }
10991 let end = chars[..i]
10992 .iter()
10993 .rposition(|c| !c.is_whitespace())
10994 .map_or(i, |p| p + 1);
10995 return Some(end);
10996 }
10997 None
10998}
10999
11000struct PrMessage {
11005 title: String,
11006 body: String,
11007}
11008
11009impl PrMessage {
11010 fn commit_message(&self) -> String {
11014 format!("{}\n\n{}", self.title, self.body)
11015 }
11016}
11017
11018fn title_marker(line: &str) -> Option<&str> {
11020 let line = line.trim();
11021 let head = line.get(..6)?;
11022 head.eq_ignore_ascii_case("title:")
11023 .then(|| line[6..].trim())
11024}
11025
11026fn summary_title(summary: &str) -> Option<String> {
11031 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11032 let raw = title_marker(first)?;
11033 if raw.is_empty() {
11034 return None;
11035 }
11036 let title = queue::title_from(raw, PR_TITLE_MAX);
11037 let lower = title.to_ascii_lowercase();
11038 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11039 return None;
11040 }
11041 Some(title)
11042}
11043
11044const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11046
11047pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11049
11050pub(crate) fn is_review_run(state: &RunState) -> bool {
11054 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11055}
11056
11057fn review_title(state: &RunState) -> Option<String> {
11062 english_subject(
11063 state.reviewed_commits.as_ref()?.first()?,
11064 state.config.graph.github_english(),
11065 )
11066}
11067
11068fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11071 let raw = raw.trim();
11072 if raw.is_empty() {
11073 return None;
11074 }
11075 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11076 return None;
11077 }
11078 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11079 return None;
11080 }
11081 let title = queue::title_from(raw, PR_TITLE_MAX);
11082 let lower = title.to_ascii_lowercase();
11083 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11084 return None;
11085 }
11086 Some(title)
11087}
11088
11089#[derive(Debug, Clone, PartialEq, Eq)]
11092struct BranchFacts {
11093 commits: Vec<(String, String)>,
11095 stat: String,
11097}
11098
11099const STAT_FILE_LINES: usize = 25;
11101const COMMITS_MAX_CHARS: usize = 20_000;
11103
11104async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11107 let commits = git::commit_log(repo, base, branch).await.ok()?;
11108 if commits.is_empty() {
11109 return None;
11110 }
11111 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11112 let lines: Vec<&str> = stat.lines().collect();
11113 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11114 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11115 let more = format!(" ... {omitted} more file(s)");
11116 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11117 kept.push(&more);
11118 kept.push(lines[lines.len() - 1]);
11119 kept.join("\n")
11120 } else {
11121 lines.join("\n")
11122 };
11123 Some(BranchFacts { commits, stat })
11124}
11125
11126fn markdown_safe(text: &str) -> String {
11129 text.replace("</details>", "</details>")
11130 .replace("\x60\x60\x60", "~~~")
11131}
11132
11133fn neutral_title(state: &RunState, winner: char) -> String {
11134 format!(
11135 "chore: land candidate {} of run {}",
11136 winner.to_ascii_uppercase(),
11137 state.id
11138 )
11139}
11140
11141pub(crate) async fn repair_landing_gate(
11144 state: &mut RunState,
11145 winner: &Candidate,
11146 outcomes: &[CommandOutcome],
11147) -> Result<GateFix> {
11148 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11149 return Ok(GateFix::Stop);
11150 }
11151 let mut runner = Runner {
11152 roles: state.config.resolve_roles()?,
11153 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11154 state: state.clone(),
11155 pause: Pause::new(),
11156 interrupt: Pause::new(),
11157 };
11158 let result = runner.gate_fix_round(winner, outcomes).await;
11159 *state = runner.state;
11160 result
11161}
11162
11163pub(crate) async fn check_gate_repair_e2e(
11166 state: &mut RunState,
11167 winner: &Candidate,
11168 wait: &crate::repo_lock::WaitPolicy,
11169) -> Result<GateFix> {
11170 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11171 return Ok(GateFix::Retry);
11172 }
11173 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11174 if state
11175 .gate_repair_e2e
11176 .as_ref()
11177 .is_some_and(|r| r.head == head && r.passed)
11178 {
11179 return Ok(GateFix::Retry);
11180 }
11181 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11182 head: head.clone(),
11183 passed: false,
11184 outcomes: Vec::new(),
11185 });
11186 state.save()?;
11187 let commands = state.config.verify.e2e.clone();
11188 let outcomes = if commands.is_empty() {
11189 Vec::new()
11190 } else {
11191 let shell = state.config.shell();
11192 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11193 let cache_dir = state.config.cache_dir();
11194 let (outcomes, _) = with_cache_lease(
11195 state,
11196 cache_dir.as_deref(),
11197 "e2e",
11198 "e2e",
11199 &winner.worktree,
11200 &head,
11201 timeout,
11202 "gate repair e2e",
11203 wait,
11204 |state, budget| async move {
11205 run_e2e_with_retry(
11206 state,
11207 &shell,
11208 &commands,
11209 &winner.worktree,
11210 budget,
11211 "gate repair e2e",
11212 )
11213 .await
11214 },
11215 )
11216 .await?;
11217 outcomes
11218 };
11219 let deferred = verify_inconclusive(&outcomes);
11220 let passed = outcomes.iter().all(CommandOutcome::ok)
11221 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11222 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11223 head,
11224 passed,
11225 outcomes,
11226 });
11227 state.save()?;
11228 if deferred {
11229 Ok(GateFix::Defer)
11230 } else if !passed {
11231 state.event(
11232 "gate",
11233 "verify.e2e failed after gate repair; nothing published",
11234 );
11235 Ok(GateFix::Stop)
11236 } else {
11237 Ok(GateFix::Retry)
11238 }
11239}
11240
11241pub(crate) async fn run_landing_gate(
11243 state: &mut RunState,
11244 winner: &Candidate,
11245 wait: &crate::repo_lock::WaitPolicy,
11246) -> Result<Vec<CommandOutcome>> {
11247 let shell = state.config.shell();
11248 let gate_commands = state.config.verify.gate.clone();
11249 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11250 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11251 Vec::new()
11252 } else {
11253 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11254 let cache_dir = state.config.cache_dir();
11255 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11256 let (outcomes, _) = with_cache_lease(
11257 state,
11258 cache_dir.as_deref(),
11259 "gate",
11260 "gate",
11261 &winner.worktree,
11262 &head,
11263 timeout,
11264 "final gate",
11265 wait,
11266 |state, budget| {
11267 let shell = shell.clone();
11268 let gate_commands = gate_commands.clone();
11269 let winner = winner.clone();
11270 async move {
11271 Runner::run_pre_gate(state, &winner).await;
11272 let (outcomes, timed_out_pids) = run_commands(
11273 state,
11274 "gate",
11275 "gate",
11276 0,
11277 &shell,
11278 &gate_commands,
11279 &winner.worktree,
11280 budget,
11281 )
11282 .await;
11283 (outcomes, false, timed_out_pids)
11284 }
11285 },
11286 )
11287 .await?;
11288 outcomes
11289 };
11290 if outcomes.is_empty() {
11291 state.event("gate", "no gate commands configured; not tested");
11296 }
11297 for o in &outcomes {
11298 state.event(
11299 "gate",
11300 format!(
11301 "`{}` -> {}",
11302 o.command,
11303 if o.ok() {
11304 "pass".to_owned()
11305 } else {
11306 format!(
11307 "FAIL ({:?})\n{}",
11308 o.code,
11309 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11310 )
11311 }
11312 ),
11313 );
11314 }
11315 Ok(outcomes)
11316}
11317
11318pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11324 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11325 ""
11326 } else {
11327 pr_title
11328 }
11329}
11330
11331pub fn landing_subject_source(state: &RunState) -> String {
11335 if is_review_run(state) {
11336 let winner = state.candidates.first().map_or('A', |c| c.label);
11337 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11338 }
11339 state.instruction.clone()
11340}
11341
11342fn summary_without_title(summary: &str) -> String {
11345 let mut lines = summary.trim().lines().peekable();
11346 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11347 lines.next();
11348 }
11349 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11350}
11351
11352#[cfg(test)]
11364fn pr_message(state: &RunState, winner: char) -> PrMessage {
11365 pr_message_with(state, winner, None)
11366}
11367
11368#[cfg(test)]
11371fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11372 let raw = pr_message_raw(
11373 state,
11374 winner,
11375 facts,
11376 crate::pr_outcome::FinalHead::Unreported,
11377 ChangeNote::NotAsked,
11378 );
11379 let id = crate::scrub::Identity::current();
11380 PrMessage {
11381 title: crate::scrub::scrub(&raw.title, &id),
11382 body: crate::scrub::scrub(&raw.body, &id),
11383 }
11384}
11385
11386#[derive(Debug, Clone, Copy)]
11388enum ChangeNote<'a> {
11389 NotAsked,
11391 Summary(&'a crate::run::ChangeSummary),
11393 Unavailable(&'a str),
11395}
11396
11397fn pr_message_raw(
11398 state: &RunState,
11399 winner: char,
11400 facts: Option<&BranchFacts>,
11401 head: crate::pr_outcome::FinalHead<'_>,
11402 change: ChangeNote<'_>,
11403) -> PrMessage {
11404 if !matches!(change, ChangeNote::NotAsked) {
11405 let (title, mut body) = match change {
11406 ChangeNote::Summary(c) => (
11407 c.title.clone(),
11408 format!("## Summary\n\n{}\n\n", c.body.trim()),
11409 ),
11410 ChangeNote::Unavailable(why) => (
11411 neutral_title(state, winner),
11412 format!(
11413 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11414 ),
11415 ),
11416 ChangeNote::NotAsked => unreachable!(),
11417 };
11418 body.push_str(&public_pr_evidence(state, head));
11419 body.push_str(&format!(
11420 "\n---\nmagi:run/{} magi:candidate-{}\n",
11421 state.id,
11422 winner.to_ascii_lowercase()
11423 ));
11424 return PrMessage { title, body };
11425 }
11426 pr_preview(state, winner, facts, head)
11427}
11428
11429fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11432 let mut body = crate::pr_outcome::review_rounds(state);
11433 body.push_str(&crate::pr_outcome::final_verification(state, head));
11434 let open = state.last_round_findings();
11435 if !open.is_empty() {
11436 body.push_str("## Open review findings\n\n");
11437 for f in open {
11438 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11439 }
11440 body.push('\n');
11441 }
11442 if let Some(c) = state.change_summary.as_ref()
11443 && head == crate::pr_outcome::FinalHead::At(&c.head)
11444 {
11445 body.push_str(&format!(
11446 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11447 c.base, c.head
11448 ));
11449 }
11450 body
11451}
11452
11453fn pr_preview(
11455 state: &RunState,
11456 winner: char,
11457 facts: Option<&BranchFacts>,
11458 head: crate::pr_outcome::FinalHead<'_>,
11459) -> PrMessage {
11460 let summary = state
11461 .candidates
11462 .iter()
11463 .find(|c| c.label == winner)
11464 .map(|c| c.summary.as_str())
11465 .unwrap_or_default();
11466 let review = is_review_run(state);
11469 let title = if review {
11470 facts
11471 .and_then(|f| {
11472 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11473 })
11474 .or_else(|| review_title(state))
11475 .or_else(|| {
11476 state
11477 .candidates
11478 .iter()
11479 .find(|c| c.label == winner)
11480 .filter(|c| !c.branch.starts_with("magi/"))
11481 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11482 })
11483 .unwrap_or_else(|| neutral_title(state, winner))
11484 } else {
11485 summary_title(summary).unwrap_or_else(|| {
11486 english_title(
11487 &state.instruction,
11488 PR_TITLE_MAX,
11489 state.config.graph.github_english(),
11490 )
11491 .unwrap_or_else(|| neutral_title(state, winner))
11492 })
11493 };
11494
11495 let mut body = String::new();
11496 let what = summary_without_title(summary);
11497 if !what.is_empty() {
11498 body.push_str("## Summary\n\n");
11499 body.push_str(&what);
11500 body.push_str("\n\n");
11501 }
11502
11503 body.push_str(&crate::pr_outcome::review_rounds(state));
11505 body.push_str(&crate::pr_outcome::final_verification(state, head));
11506 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11507
11508 let open = state.open_findings();
11509 if !open.is_empty() {
11510 body.push_str("## Open review findings\n\n");
11511 for f in &open {
11512 body.push_str(&format!(
11513 "- `{}` [{:?}] {}\n",
11514 f.id, f.severity, "Context retained in local review records."
11515 ));
11516 }
11517 body.push('\n');
11518 }
11519
11520 if let Some(fix) = fix
11521 && !fix.rejected.is_empty()
11522 {
11523 body.push_str("## Declined by the fixer\n\n");
11524 for r in &fix.rejected {
11525 body.push_str(&format!(
11526 "- `{}`: {}\n",
11527 r.id, "Reason retained in local review records."
11528 ));
11529 }
11530 body.push('\n');
11531 }
11532
11533 if review {
11534 body.push_str("## Commits under review\n\n");
11536 if let Some(facts) = facts {
11537 let mut left = COMMITS_MAX_CHARS;
11538 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11539 let mut entry = format!("- {}\n", markdown_safe(subject));
11540 for l in markdown_safe(text).lines() {
11541 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11542 }
11543 if left == 0 {
11544 body.push_str(&format!(
11545 "- ... {} more commit(s)\n",
11546 facts.commits.len() - i
11547 ));
11548 break;
11549 }
11550 if entry.len() > left {
11551 let mut end = left;
11554 while !entry.is_char_boundary(end) {
11555 end -= 1;
11556 }
11557 entry.truncate(end);
11558 entry.push_str("\n ... (truncated)\n");
11559 left = 0;
11560 } else {
11561 left -= entry.len();
11562 }
11563 body.push_str(&entry);
11564 }
11565 if !facts.stat.trim().is_empty() {
11566 body.push_str(&format!(
11567 "\n## Diff stat\n\n```\n{}\n```\n",
11568 markdown_safe(facts.stat.trim())
11569 ));
11570 }
11571 } else {
11572 match &state.reviewed_commits {
11573 Some(subjects) => {
11574 for s in subjects {
11575 body.push_str(&format!("- {}\n", s.trim()));
11576 }
11577 }
11578 None => {
11579 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11582 body.push_str(rest.trim());
11583 body.push('\n');
11584 }
11585 }
11586 }
11587 } else {
11588 let task = state.instruction.trim();
11589 let task = if task.is_empty() {
11590 "(empty task)"
11591 } else {
11592 task
11593 };
11594 body.push_str(&format!(
11595 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11596 task.replace("</details>", "</details>")
11597 ));
11598 }
11599
11600 body.push_str(&format!(
11601 "\n---\nmagi:run/{} magi:candidate-{}\n",
11602 state.id,
11603 winner.to_ascii_lowercase()
11604 ));
11605
11606 PrMessage { title, body }
11607}
11608
11609fn seeded_instruction(state: &RunState) -> String {
11612 let mut inputs = Vec::new();
11613 if let Some(start) = &state.sources.start_from {
11614 inputs.push(format!(
11615 "Start from `{}` at `{}` (including ancestry).",
11616 start.token, start.sha
11617 ));
11618 }
11619 for pick in &state.sources.pick_commits {
11620 inputs.push(format!(
11621 "Pick `{}` at `{}` (single commit).",
11622 pick.token, pick.sha
11623 ));
11624 }
11625 if inputs.is_empty() {
11626 state.instruction.clone()
11627 } else {
11628 format!(
11629 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11630 state.instruction,
11631 inputs.join("\n")
11632 )
11633 }
11634}
11635
11636async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11640 let base = &state.base_branch;
11641 let mut against = base.clone();
11642 if mode == MergeMode::Pr {
11643 let remote = &state.config.merge.remote;
11647 let tracking = format!("{remote}/{base}");
11648 let fetched = git::fetch(repo, remote, base).await;
11649 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11650 against = tracking;
11651 } else {
11652 return false;
11653 }
11654 }
11655 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11656}
11657
11658fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11661 let mut detail = format!(
11662 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11663 nothing to open a pull request for"
11664 );
11665 detail.push_str(
11666 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11667 );
11668 detail
11669}
11670
11671#[derive(Debug, PartialEq, Eq)]
11674enum PrPlan {
11675 Create,
11676 Adopt { url: String, title: String },
11677 Stop(String),
11678}
11679
11680fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11683 match found {
11684 Ok(land::OpenPr::None) => PrPlan::Create,
11685 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11686 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11687 "several open pull requests exist for this branch, not picking one: {}",
11688 urls.join(" ")
11689 )),
11690 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11691 }
11692}
11693
11694pub(crate) async fn gh_pr_create(
11696 cwd: &Path,
11697 base: &str,
11698 head: &str,
11699 title: &str,
11700 body: &str,
11701) -> Result<String> {
11702 let out = tokio::process::Command::new("gh")
11703 .args([
11704 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11705 ])
11706 .current_dir(cwd)
11707 .quiet()
11708 .stdin(std::process::Stdio::null())
11709 .output()
11710 .await
11711 .context("spawn gh")?;
11712 if out.status.success() {
11713 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11714 } else {
11715 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11716 }
11717}
11718
11719async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11721 match std::fs::symlink_metadata(path) {
11722 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11723 git::worktree_remove_clean(repo, path).await?;
11726 return Ok(());
11727 }
11728 Err(e) => return Err(e.into()),
11729 Ok(_) => {}
11730 }
11731 if !git::is_clean(path).await? {
11732 bail!(
11733 "worktree has uncommitted work; refusing removal: {}",
11734 path.display()
11735 );
11736 }
11737 if !git::worktree_remove_clean(repo, path).await? {
11738 bail!("worktree removal refused: {}", path.display());
11739 }
11740 Ok(())
11741}
11742
11743pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11752 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11753 bail!("refusing fold: {reason}");
11754 }
11755 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11756 bail!("candidate preservation incomplete; refusing cleanup");
11757 }
11758 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11762 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11763 let _driver = crate::driver::maintenance(home, &state.id)?;
11764 let mut output_guards = Vec::new();
11765 for entry in std::fs::read_dir(state.worktree_root())
11766 .into_iter()
11767 .flatten()
11768 .flatten()
11769 {
11770 if entry.path().join(".git").is_file() {
11771 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11772 }
11773 }
11774 let repo = state.repo.clone();
11775 let wait = repository_wait(state, None);
11778 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11779 Ok(lease) => lease,
11780 Err(e) => {
11781 state.save_under(home)?;
11782 return Err(e);
11783 }
11784 };
11785 let _source_outputs = if state.status == RunStatus::Merged
11788 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11789 {
11790 verify_merged_fold_sources(state, drop_winner, home).await?
11791 } else {
11792 Vec::new()
11793 };
11794 let root = state.worktree_root();
11795 let winner = state.tally.as_ref().map(|t| t.winner);
11796 let mut removed = Vec::new();
11797
11798 for i in 0..state.candidates.len() {
11799 let c = state.candidates[i].clone();
11800 let is_winner = Some(c.label) == winner;
11801 if is_winner && !drop_winner {
11802 continue;
11803 }
11804 let existed = c.worktree.exists();
11805 remove_preserved_worktree(&repo, &c.worktree).await?;
11808 if existed {
11809 removed.push(c.worktree.to_string_lossy().into_owned());
11810 }
11811 let handed_over = state.released_branches.contains(&c.branch);
11814 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11815 if !git::branch_delete(&repo, &c.branch).await? {
11816 bail!("candidate branch removal refused");
11817 }
11818 removed.push(c.branch.clone());
11819 }
11820 state.candidates[i].folded = true;
11821 }
11822
11823 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11824 let path = name.path();
11825 let keep = !drop_winner
11826 && winner.is_some_and(|w| {
11827 path.file_name()
11828 .is_some_and(|n| n == format!("cand-{w}").as_str())
11829 });
11830 if keep {
11831 continue;
11832 }
11833 remove_preserved_worktree(&repo, &path).await?;
11834 removed.push(path.to_string_lossy().into_owned());
11835 }
11836
11837 remove_if_empty(&root);
11846
11847 if state.enabled_worktree_config && drop_winner {
11848 git::release_worktree_config(&repo).await.ok();
11852 state.enabled_worktree_config = false;
11853 }
11854 state.save_under(home)?;
11855 Ok(removed)
11856}
11857
11858async fn verify_merged_fold_sources(
11859 state: &RunState,
11860 drop_winner: bool,
11861 home: &Path,
11862) -> Result<Vec<std::fs::File>> {
11863 let winner = state.tally.as_ref().map(|t| t.winner);
11864 let mut heads = BTreeSet::new();
11865 for candidate in &state.candidates {
11866 if Some(candidate.label) == winner && !drop_winner {
11867 continue;
11868 }
11869 anyhow::ensure!(
11870 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11871 "candidate recovery source is outside the inventoried run root: {}",
11872 candidate.worktree.display()
11873 );
11874 match std::fs::symlink_metadata(&candidate.worktree) {
11875 Ok(_) => {}
11876 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11877 Err(error) => return Err(error.into()),
11878 }
11879 if !state.released_branches.contains(&candidate.branch)
11880 && git::branch_exists(&state.repo, &candidate.branch).await?
11881 {
11882 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11883 }
11884 }
11885 let mut cargo_guards = Vec::new();
11886 let entries = match std::fs::read_dir(state.worktree_root()) {
11887 Ok(entries) => Some(entries),
11888 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11889 Err(error) => return Err(error.into()),
11890 };
11891 for entry in entries.into_iter().flatten() {
11892 let path = entry?.path();
11893 let metadata = std::fs::symlink_metadata(&path)?;
11894 anyhow::ensure!(
11895 metadata.is_dir() && !metadata.file_type().is_symlink(),
11896 "non-directory or symlink recovery source retained: {}",
11897 path.display()
11898 );
11899 if !drop_winner
11900 && winner.is_some_and(|w| {
11901 path.file_name()
11902 .is_some_and(|name| name == format!("cand-{w}").as_str())
11903 })
11904 {
11905 continue;
11906 }
11907 anyhow::ensure!(
11908 path.join(".git").is_file(),
11909 "unregistered recovery source retained: {}",
11910 path.display()
11911 );
11912 anyhow::ensure!(
11913 git::git(
11914 &path,
11915 &[
11916 "status",
11917 "--porcelain",
11918 "--untracked-files=all",
11919 "--ignored",
11920 "--",
11921 ".",
11922 ":(exclude)target"
11923 ]
11924 )
11925 .await?
11926 .is_empty(),
11927 "source changes or ignored recovery files retained: {}",
11928 path.display()
11929 );
11930 match std::fs::symlink_metadata(path.join("target")) {
11931 Ok(_) => {
11932 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11933 }
11934 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11935 Err(error) => return Err(error.into()),
11936 }
11937 for checkpoint in [
11938 "rebase-merge",
11939 "rebase-apply",
11940 "sequencer",
11941 "CHERRY_PICK_HEAD",
11942 "MERGE_HEAD",
11943 "REBASE_HEAD",
11944 "REVERT_HEAD",
11945 "BISECT_LOG",
11946 "AUTO_MERGE",
11947 "MERGE_MSG",
11948 ] {
11949 let location = git::git(
11950 &path,
11951 &[
11952 "rev-parse",
11953 "--path-format=absolute",
11954 "--git-path",
11955 checkpoint,
11956 ],
11957 )
11958 .await?;
11959 anyhow::ensure!(
11960 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11961 if e.kind() == std::io::ErrorKind::NotFound),
11962 "Git recovery checkpoint retained: {checkpoint}"
11963 );
11964 }
11965 heads.insert(git::rev_parse(&path, "HEAD").await?);
11966 }
11967 crate::fold_evidence::preserve(state, home, &heads).await?;
11968
11969 Ok(cargo_guards)
11970}
11971
11972fn remove_if_empty(dir: &Path) {
11983 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11984 std::fs::remove_dir(dir).ok();
11985 }
11986}
11987
11988pub fn worst_open(state: &RunState) -> Option<Severity> {
11990 state
11991 .reviews
11992 .last()?
11993 .reviews
11994 .iter()
11995 .flat_map(|r| r.findings.iter())
11996 .map(|f| f.severity)
11997 .max()
11998}
11999
12000async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12004 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12005 git::acquire_worktree_config(repo).await?;
12006 state.enabled_worktree_config = true;
12007 }
12008 Ok(())
12009}
12010
12011#[cfg(test)]
12012mod tests {
12013 #[tokio::test]
12014 async fn launcher_base_fetch_requires_remote_success() {
12015 let tmp = tempfile::tempdir().unwrap();
12016 let repo = tmp.path().join("repo");
12017 let remote = tmp.path().join("remote");
12018 std::fs::create_dir(&repo).unwrap();
12019 std::fs::create_dir(&remote).unwrap();
12020 git::git(&remote, &["init", "--bare", "-b", "main"])
12021 .await
12022 .unwrap();
12023 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12024 git::git(
12025 &repo,
12026 &[
12027 "-c",
12028 "user.name=Fixture",
12029 "-c",
12030 "user.email=fixture@example.invalid",
12031 "commit",
12032 "--allow-empty",
12033 "-m",
12034 "fixture",
12035 ],
12036 )
12037 .await
12038 .unwrap();
12039 git::git(
12040 &repo,
12041 &["remote", "add", "origin", remote.to_str().unwrap()],
12042 )
12043 .await
12044 .unwrap();
12045 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12046 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12047 assert_eq!(
12048 resolve_base(&repo, "main", "origin", None)
12049 .await
12050 .unwrap()
12051 .commit,
12052 head
12053 );
12054 #[cfg(unix)]
12055 {
12056 let policy: crate::config::Coordination = toml::from_str(
12057 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12058 ).unwrap();
12059 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12060 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12061 let held = crate::repo_lock::SeatLease::operation_with(
12062 &repo,
12063 Some(&policy),
12064 &crate::repo_lock::WaitPolicy::default(),
12065 )
12066 .await
12067 .unwrap();
12068 assert!(held.is_some());
12069 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12070 tokio::pin!(fetch);
12071 assert!(
12072 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12073 .await
12074 .is_err()
12075 );
12076 assert!(
12077 !repo.join(".git/FETCH_HEAD").exists(),
12078 "no fetch before admission"
12079 );
12080 drop(held);
12081 let fetched = fetch.await.unwrap();
12082 assert_eq!(fetched.commit, head);
12083 assert_eq!(fetched.remote, "origin");
12084 assert_eq!(fetched.branch, "main");
12085 }
12086 git::git(
12088 &repo,
12089 &[
12090 "remote",
12091 "set-url",
12092 "origin",
12093 tmp.path().join("missing").to_str().unwrap(),
12094 ],
12095 )
12096 .await
12097 .unwrap();
12098 let error = resolve_base(&repo, "main", "origin", None)
12099 .await
12100 .unwrap_err()
12101 .to_string();
12102 assert!(error.contains("cannot read origin/main"));
12103 assert!(error.contains("refusing to branch off"));
12104 }
12105
12106 #[test]
12107 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12108 let leaked = vec!["chore(deps): update a crate".to_owned()];
12109 let own = "fix(daemon): apply a chosen action";
12110 assert!(should_retitle("", own, &leaked));
12111 assert!(should_retitle(
12112 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12113 own,
12114 &leaked
12115 ));
12116 assert!(should_retitle(
12117 "chore: land candidate A of run 1",
12118 own,
12119 &leaked
12120 ));
12121 assert!(should_retitle(
12122 "magi: candidate A (uncommitted work)",
12123 own,
12124 &leaked
12125 ));
12126 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12127 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12128 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12129 }
12130
12131 #[test]
12132 fn pr_merge_plan_creates_adopts_or_stops() {
12133 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12134 assert_eq!(
12135 pr_merge_plan(Ok(land::OpenPr::One {
12136 url: "u".into(),
12137 title: "t".into()
12138 })),
12139 PrPlan::Adopt {
12140 url: "u".into(),
12141 title: "t".into()
12142 }
12143 );
12144 let PrPlan::Stop(many) =
12145 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12146 else {
12147 panic!("many must stop");
12148 };
12149 assert!(many.contains('a') && many.contains('b'));
12150 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12151 panic!("a failed lookup must stop");
12152 };
12153 assert!(err.contains("bad token"));
12154 }
12155
12156 use super::*;
12157 use crate::run::GateStatus;
12158 use std::collections::BTreeMap;
12159 use std::time::Duration;
12160
12161 fn conductor() -> AgentSpec {
12162 AgentSpec {
12163 id: "conductor".to_owned(),
12164 kind: crate::config::AgentKind::Command,
12165 model: None,
12166 command: vec!["true".to_owned()],
12167 extra_args: Vec::new(),
12168 env: BTreeMap::new(),
12169 prompt_delivery: None,
12170 max_turn_secs: None,
12171 }
12172 }
12173
12174 fn spec(id: &str) -> AgentSpec {
12175 AgentSpec {
12176 id: id.to_owned(),
12177 kind: crate::config::AgentKind::Command,
12178 model: None,
12179 command: vec!["true".to_owned()],
12180 extra_args: Vec::new(),
12181 env: BTreeMap::new(),
12182 prompt_delivery: None,
12183 max_turn_secs: None,
12184 }
12185 }
12186
12187 fn ids(xs: &[&str]) -> BTreeSet<String> {
12188 xs.iter().map(|s| (*s).to_owned()).collect()
12189 }
12190
12191 #[test]
12192 fn pick_successor_skips_an_agent_another_seat_holds() {
12193 let roster = [spec("a"), spec("b"), spec("c")];
12195 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12196 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12197 }
12198
12199 #[test]
12200 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12201 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12203 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12204 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12205 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12206 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12207 }
12208
12209 #[test]
12210 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12211 let roster = [spec("a"), spec("b")];
12212 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12213 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12214 let carried = ids(&["b"]);
12215 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12216 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12217 }
12218
12219 #[test]
12220 fn pick_successor_returns_none_without_any_untried_successor() {
12221 let roster = [spec("a"), spec("b")];
12222 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12223 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12224 let carried = ids(&["a"]);
12225 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12226 }
12227
12228 #[test]
12229 fn pick_successor_rescue_avoids_another_seats_occupant() {
12230 let roster = [spec("a"), spec("b"), spec("c")];
12232 let carried = ids(&["b", "c"]);
12233 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12234 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12235 }
12236
12237 #[test]
12238 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12239 let roster = [spec("a"), spec("b"), spec("c")];
12240 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12241 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12242 }
12243
12244 #[test]
12245 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12246 let roster = [spec("a"), spec("b"), spec("c")];
12247 let failed = ids(&["a", "b", "c"]);
12248 let mut tried = ids(&["b"]);
12250 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12251 assert_eq!(first.id, "a");
12252 tried.insert(first.id.clone());
12253 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12254 assert_eq!(second.id, "c");
12255 tried.insert(second.id.clone());
12256 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12257 }
12258
12259 #[test]
12260 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12261 let roster = [spec("a"), spec("b")];
12262 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12263 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12264 }
12265
12266 #[test]
12267 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12268 let roster = [spec("a"), spec("b"), spec("c")];
12269 let h = SeatHistory {
12270 failed: ids(&["a"]),
12271 last_ok: Some("b".to_owned()),
12272 last_fail: None,
12273 };
12274 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12275 let gone = SeatHistory {
12277 last_ok: Some("zzz".to_owned()),
12278 ..h.clone()
12279 };
12280 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12281 let failed = SeatHistory {
12282 failed: ids(&["a", "b"]),
12283 last_ok: Some("b".to_owned()),
12284 last_fail: None,
12285 };
12286 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12287 }
12288
12289 #[test]
12290 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12291 let first = SeatState::new("review-1", "alpha", 7);
12292 let next = handover_seat("review-1", "gamma", 7);
12293 assert_ne!(first.claude_session, next.claude_session);
12294 }
12295
12296 #[test]
12297 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12298 let mut state = state_with_summary("x", "y");
12299 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12300 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12301 assert_ne!(a.claude_session, b.claude_session);
12302 }
12303
12304 #[test]
12305 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12306 let roster = [spec("a"), spec("b")];
12307 let h = SeatHistory {
12308 failed: ids(&["a", "b"]),
12309 ..SeatHistory::default()
12310 };
12311 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12312 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12313 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12314 }
12315
12316 #[test]
12323 fn failure_signature_ignores_numbers_and_paths() {
12324 assert_eq!(
12325 failure_signature("exited with Some(2) and no usable output"),
12326 failure_signature("exited with Some(137) and no usable output")
12327 );
12328 assert_eq!(
12329 failure_signature("process exited with no usable output (exit code 2)"),
12330 failure_signature("process exited with no usable output (exit code 137)")
12331 );
12332 for (old, new) in [
12333 (
12334 "exited with Some(2) and no usable output",
12335 "process exited with no usable output (exit code 2)",
12336 ),
12337 (
12338 "exited with None and no usable output",
12339 "process exited with no usable output (exit code unavailable)",
12340 ),
12341 ] {
12342 let previous = FailClass::Other(failure_signature(old));
12343 let current = FailClass::Other(failure_signature(new));
12344 assert_eq!(previous, current, "resumption retains the failure class");
12345 assert!(!should_hand_over(Some(&previous), ¤t));
12346 }
12347 assert_eq!(
12348 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12349 failure_signature("cannot open /var/x.txt: denied")
12350 );
12351 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12352 }
12353
12354 #[test]
12355 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12356 let other = FailClass::Other("x".into());
12357 assert!(should_hand_over(None, &FailClass::Quota));
12358 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12359 assert!(should_hand_over(
12360 Some(&FailClass::Timeout),
12361 &FailClass::Timeout
12362 ));
12363 assert!(should_hand_over(None, &other));
12364 assert!(!should_hand_over(Some(&other), &other));
12365 assert!(should_hand_over(
12366 Some(&other),
12367 &FailClass::Other("y".into())
12368 ));
12369 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12371 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12372 }
12373
12374 #[test]
12375 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12376 let a = SeatState::new("judge-1", "alpha", 7);
12377 let b = handover_seat("judge-1", "beta", 7);
12378 assert_ne!(a.claude_session, b.claude_session);
12379 assert_eq!(b.turns, 0);
12380 }
12381
12382 #[test]
12383 fn a_timeout_is_classified_apart_from_other_failures() {
12384 assert_eq!(
12385 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12386 Some(FailClass::Timeout)
12387 );
12388 assert!(matches!(
12389 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12390 Some(FailClass::Other(_))
12391 ));
12392 }
12393
12394 #[test]
12395 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12396 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12397 let tried = BTreeSet::from(["beta".to_owned()]);
12398 let next = next_untried_in_roster(&roster, 1, &tried);
12401 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12402 }
12403
12404 #[test]
12405 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12406 let roster = vec![spec("alpha"), spec("beta")];
12407 let tried = BTreeSet::from(["beta".to_owned()]);
12408 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12412 }
12413
12414 #[test]
12415 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12416 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12417 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12418 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12422 }
12423
12424 #[test]
12425 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12426 let roster = vec![spec("a"), spec("a"), spec("b")];
12427 let tried = BTreeSet::from(["a".to_owned()]);
12428 let next = next_untried_in_roster(&roster, 0, &tried);
12429 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12430 }
12431
12432 #[test]
12433 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12434 let roster = vec![spec("a"), spec("b")];
12435 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12436 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12437 }
12438
12439 #[test]
12440 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12441 let dir = tempfile::tempdir().unwrap();
12442 let bay = dir.path().join("ffff");
12443
12444 remove_if_empty(&bay);
12446 assert!(!bay.exists());
12447
12448 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12451 remove_if_empty(&bay);
12452 assert!(bay.exists(), "non-empty directory must survive");
12453
12454 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12456 remove_if_empty(&bay);
12457 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12458 }
12459
12460 #[test]
12469 fn a_full_panel_that_found_nothing_is_clean() {
12470 assert!(round_is_clean(
12471 0,
12472 true,
12473 2,
12474 2,
12475 0,
12476 IncompleteReviewPolicy::Block
12477 ));
12478 }
12479
12480 #[test]
12481 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12482 assert!(!round_is_clean(
12483 0,
12484 true,
12485 1,
12486 2,
12487 0,
12488 IncompleteReviewPolicy::Block
12489 ));
12490 }
12491
12492 #[test]
12493 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12494 assert!(!round_is_clean(
12495 1,
12496 true,
12497 1,
12498 2,
12499 0,
12500 IncompleteReviewPolicy::Warn
12501 ));
12502 }
12503
12504 #[test]
12505 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12506 assert!(round_is_clean(
12507 0,
12508 true,
12509 1,
12510 2,
12511 0,
12512 IncompleteReviewPolicy::Warn
12513 ));
12514 }
12515
12516 #[test]
12517 fn a_full_panel_with_an_open_finding_is_not_clean() {
12518 assert!(!round_is_clean(
12519 1,
12520 true,
12521 2,
12522 2,
12523 0,
12524 IncompleteReviewPolicy::Block
12525 ));
12526 }
12527
12528 #[test]
12529 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12530 assert!(!round_is_clean(
12531 0,
12532 false,
12533 2,
12534 2,
12535 0,
12536 IncompleteReviewPolicy::Block
12537 ));
12538 }
12539
12540 #[test]
12547 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12548 assert!(round_is_clean(
12551 0,
12552 true,
12553 1,
12554 2,
12555 1,
12556 IncompleteReviewPolicy::Block
12557 ));
12558 }
12559
12560 #[test]
12561 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12562 assert!(!round_is_clean(
12565 0,
12566 true,
12567 1,
12568 2,
12569 0,
12570 IncompleteReviewPolicy::Block
12571 ));
12572 }
12573
12574 #[test]
12575 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12576 assert!(!round_is_clean(
12577 1,
12578 true,
12579 1,
12580 2,
12581 1,
12582 IncompleteReviewPolicy::Block
12583 ));
12584 assert!(!round_is_clean(
12585 0,
12586 false,
12587 1,
12588 2,
12589 1,
12590 IncompleteReviewPolicy::Block
12591 ));
12592 }
12593
12594 #[test]
12595 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12596 assert!(!round_is_clean(
12600 0,
12601 true,
12602 0,
12603 2,
12604 2,
12605 IncompleteReviewPolicy::Block
12606 ));
12607 }
12608
12609 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12610 CommandOutcome {
12611 command: "test".to_owned(),
12612 code,
12613 output_tail: String::new(),
12614 duration_ms: 0,
12615 resource_blocked,
12616 }
12617 }
12618
12619 #[test]
12620 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12621 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12622 assert!(
12623 !verify_inconclusive(&[outcome(Some(1), false)]),
12624 "an ordinary failure is still evidence about the patch"
12625 );
12626 assert!(verify_inconclusive(&[outcome(None, true)]));
12627 assert!(
12628 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12629 "one inconclusive outcome taints the whole batch"
12630 );
12631 assert!(!verify_inconclusive(&[]));
12632 }
12633
12634 #[tokio::test]
12635 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12636 let calls = std::sync::atomic::AtomicUsize::new(0);
12640 let started = Instant::now();
12641 wait_for_pids_with(
12642 &[123],
12643 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12644 Duration::from_millis(5),
12645 Duration::from_secs(5),
12646 )
12647 .await;
12648 assert!(
12649 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12650 "must keep checking rather than deciding on the first answer"
12651 );
12652 assert!(
12653 started.elapsed() < Duration::from_secs(1),
12654 "must return the moment it is confirmed dead, not wait out the ceiling"
12655 );
12656 }
12657
12658 #[tokio::test]
12659 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12660 let started = Instant::now();
12661 wait_for_pids_with(
12662 &[123],
12663 |_| true, Duration::from_millis(5),
12665 Duration::from_millis(30),
12666 )
12667 .await;
12668 let elapsed = started.elapsed();
12669 assert!(
12670 elapsed >= Duration::from_millis(30),
12671 "must not give up before its own ceiling: {elapsed:?}"
12672 );
12673 assert!(
12674 elapsed < Duration::from_secs(1),
12675 "must not wait past its own ceiling either: {elapsed:?}"
12676 );
12677 }
12678
12679 #[tokio::test]
12680 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12681 let started = Instant::now();
12682 wait_for_pids_with(
12683 &[],
12684 |_| true,
12685 Duration::from_secs(5),
12686 Duration::from_secs(5),
12687 )
12688 .await;
12689 assert!(
12690 started.elapsed() < Duration::from_millis(200),
12691 "an empty pid list has nothing to confirm"
12692 );
12693 }
12694
12695 fn review_round(
12701 clean: bool,
12702 blocking: usize,
12703 answered: usize,
12704 expected: usize,
12705 progressed: bool,
12706 e2e_ok: bool,
12707 ) -> ReviewRound {
12708 ReviewRound {
12709 round: 1,
12710 head: "h".to_owned(),
12711 verified_head: None,
12712 verified_at: None,
12713 reviews: Vec::new(),
12714 e2e: vec![CommandOutcome {
12715 command: "test".to_owned(),
12716 code: Some(if e2e_ok { 0 } else { 1 }),
12717 output_tail: String::new(),
12718 duration_ms: 0,
12719 resource_blocked: false,
12720 }],
12721 verify_retried: false,
12722 e2e_deferred: false,
12723 e2e_defer_reason: None,
12724 fix: None,
12725 blocking,
12726 answered,
12727 expected,
12728 clean,
12729 progressed,
12730 vote_split: false,
12731 reconsideration: Vec::new(),
12732 verdict: None,
12733 }
12734 }
12735
12736 #[test]
12737 fn review_conclusion_is_none_when_nothing_has_run() {
12738 assert_eq!(review_conclusion(&[], 3), None);
12739 }
12740
12741 #[test]
12742 fn review_conclusion_is_none_while_rounds_remain() {
12743 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12744 assert_eq!(review_conclusion(&rounds, 3), None);
12745 }
12746
12747 #[test]
12748 fn review_conclusion_is_gating_once_a_round_is_clean() {
12749 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12750 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12751 }
12752
12753 #[test]
12754 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12755 let rounds = vec![
12756 review_round(false, 1, 2, 2, true, true),
12757 review_round(false, 1, 2, 2, true, true),
12758 ];
12759 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12760 }
12761
12762 #[test]
12763 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12764 let rounds = vec![
12765 review_round(false, 1, 2, 2, true, true),
12766 review_round(false, 1, 2, 2, true, false),
12767 ];
12768 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12769 }
12770
12771 #[test]
12772 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12773 let mut blocked = review_round(false, 1, 2, 2, true, false);
12780 blocked.e2e[0].resource_blocked = true;
12781 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12782 assert_eq!(review_conclusion(&rounds, 2), None);
12783 }
12784
12785 #[test]
12786 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12787 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12789 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12790 }
12791
12792 #[test]
12793 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12794 let rounds = vec![
12795 review_round(false, 1, 2, 2, false, true),
12796 review_round(false, 1, 2, 2, false, true),
12797 ];
12798 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12799 }
12800
12801 fn secs(n: u64) -> Duration {
12802 Duration::from_secs(n)
12803 }
12804
12805 #[tokio::test]
12808 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12809 ask_test_home();
12810 let tmp = tempfile::tempdir().unwrap();
12811 let repo = tmp.path().join("repo");
12812 std::fs::create_dir_all(&repo).unwrap();
12813 init_repo(&repo);
12814 let g = |args: &[&str]| {
12815 let out = std::process::Command::new("git")
12816 .args(args)
12817 .current_dir(&repo)
12818 .quiet()
12819 .output()
12820 .unwrap();
12821 assert!(
12822 out.status.success(),
12823 "{}",
12824 String::from_utf8_lossy(&out.stderr)
12825 );
12826 };
12827 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12828 g(&["checkout", "-b", "historical"]);
12829 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12830 g(&["add", "."]);
12831 g(&["commit", "-m", "old alternative"]);
12832 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12833 g(&["checkout", "-b", "magi/task-selected", &base]);
12834 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12835 g(&["add", "."]);
12836 g(&["commit", "-m", "measurement infrastructure"]);
12837 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12838 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12839 g(&["add", "."]);
12840 g(&["commit", "-m", "dependent implementation"]);
12841 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12842 g(&["checkout", "main"]);
12843 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12844 g(&["add", "."]);
12845 g(&["commit", "-m", "unrelated base change"]);
12846 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12847 let origin = tmp.path().join("origin.git");
12848 g(&[
12849 "clone",
12850 "--bare",
12851 repo.to_str().unwrap(),
12852 origin.to_str().unwrap(),
12853 ]);
12854 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12855 g(&["fetch", "origin"]);
12856 let selection = crate::pr_recovery::Selection {
12857 url: "https://example.test/project/pull/1".into(),
12858 branch: "magi/task-selected".into(),
12859 head: head.clone(),
12860 source_head: None,
12861 source_run: None,
12862 source_proof: None,
12863 disposition: Some(crate::pr_recovery::Disposition::Review),
12864 instruction: "preserve measured behavior".into(),
12865 approved_at: jiff::Timestamp::now(),
12866 };
12867 let mut template = runner_at(RunStatus::Reviewing);
12868 template.state.repo = repo.clone();
12869 template.state.base_commit = current.clone();
12870 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12871 template.state.origin = Some(crate::run::Origin::queue("selected"));
12872 template.state.pr_head = Some(selection.branch.clone());
12873 template.state.pr_recovery = Some(selection.clone());
12874 let mut runner =
12875 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12876 .await
12877 .unwrap();
12878 let wt = runner.state.winner().unwrap().worktree.clone();
12879 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12880 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12881 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12882 assert!(runner.state.seeds.is_empty());
12883 assert_eq!(runner.state.instruction, "preserve measured behavior");
12884 runner.sync_to_base().await.unwrap();
12885 assert_eq!(
12886 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12887 "prerequisite"
12888 );
12889 assert_eq!(
12890 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12891 "measured implementation"
12892 );
12893 assert_eq!(
12894 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12895 "current main change"
12896 );
12897 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12898 assert_eq!(
12899 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12900 head
12901 );
12902
12903 let mut partial = runner_at(RunStatus::Reviewing);
12905 partial.state.repo = repo.clone();
12906 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12907 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12908 run: Some("exit 1".into()),
12909 ..Default::default()
12910 }];
12911 let selection = runner.state.pr_recovery.clone().unwrap();
12912 partial.state.pr_recovery = Some(selection.clone());
12913 partial.state.pr_head = Some(selection.branch.clone());
12914 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12915 let id = partial.state.id.clone();
12916 let error = match Runner::open_selected(
12917 &repo,
12918 partial.state,
12919 partial.roles,
12920 1,
12921 runner.state.base_commit.clone(),
12922 selection,
12923 )
12924 .await
12925 {
12926 Ok(_) => panic!("setup must fail"),
12927 Err(e) => e,
12928 };
12929 assert_eq!(
12930 error
12931 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12932 .unwrap()
12933 .run,
12934 id
12935 );
12936 let failed = RunState::load(&id).unwrap();
12937 assert_eq!(failed.status, RunStatus::Blocked);
12938 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12939 assert!(Runner::resume(&id).is_err());
12940 runner.state.status = RunStatus::Ready;
12941 runner.state.driver_exited = true;
12942 runner.state.save().unwrap();
12943 let mut retry = runner_at(RunStatus::Reviewing);
12944 retry.state.repo = repo.clone();
12945 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12946 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12947 reselected.approved_at = jiff::Timestamp::now();
12948 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12949 retry.state.pr_head = Some(reselected.branch.clone());
12950 retry.state.pr_recovery = Some(reselected.clone());
12951 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12952 .await
12953 .unwrap();
12954 let restarted = Runner::open_selected(
12955 &repo,
12956 retry.state,
12957 retry.roles,
12958 1,
12959 runner.state.base_commit.clone(),
12960 reselected,
12961 )
12962 .await
12963 .unwrap();
12964 assert_eq!(
12965 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12966 .await
12967 .unwrap(),
12968 head
12969 );
12970 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12971 assert_eq!(
12972 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12973 head
12974 );
12975 }
12976
12977 #[tokio::test]
12978 #[cfg(unix)]
12979 async fn fallback_and_interrupted_restoration_require_coordination() {
12980 use std::os::fd::AsRawFd as _;
12981 ask_test_home();
12982 let tmp = tempfile::tempdir().unwrap();
12983 let mut runner = registered_runner(tmp.path()).await;
12984 runner.state.config.graph.timeout_resource_wait = 1;
12985 let winner = runner.state.candidates[0].clone();
12986 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12987 let leftover = winner.worktree.join("leftover");
12988 std::fs::write(&leftover, "untrusted edits").unwrap();
12989 let next = tmp.path().join("next-fixer");
12990 let mut failed = spec("first");
12991 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12992 let mut successor = spec("second");
12993 successor.command = vec![
12994 "sh".into(),
12995 "-c".into(),
12996 format!("touch '{}'; echo done", next.display()),
12997 ];
12998 let attempts = vec![
12999 (failed.clone(), "first".into()),
13000 (successor.clone(), "second".into()),
13001 ];
13002 let lock_path = runner.state.repo.join(".git/updater.lock");
13003 let external = std::fs::File::create(&lock_path).unwrap();
13004 assert_eq!(
13005 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13006 0
13007 );
13008 let artifacts = tmp.path().join("artifacts");
13009 let build = |spec, seat| SeatJob {
13010 spec,
13011 seat,
13012 cwd: winner.worktree.clone(),
13013 prompt: "fixture".into(),
13014 timeout: Duration::from_secs(10),
13015 allow_write: true,
13016 sessions: false,
13017 artifacts: artifacts.clone(),
13018 stem: "fix".into(),
13019 handover: None,
13020 };
13021 let (_, _, out) = runner
13022 .ask_fixer_among(
13023 attempts.clone(),
13024 "graft",
13025 None,
13026 Some((&winner.worktree, &head)),
13027 build,
13028 )
13029 .await;
13030 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13031 assert!(leftover.exists(), "contention must prevent clean");
13032 assert!(!next.exists());
13033 assert!(runner.state.handovers.is_empty());
13034 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13036 from_run: "earlier".into(),
13037 generation: 1,
13038 mode: crate::recovery::Mode::VerifyRepair,
13039 reasons: Vec::new(),
13040 reviewed_head: None,
13041 rounds: Vec::new(),
13042 open_findings: Vec::new(),
13043 evidence: None,
13044 criteria_inherited: true,
13045 repair: crate::recovery::Repair::Started {
13046 before: head.clone(),
13047 },
13048 });
13049 assert!(
13050 runner
13051 .recovery_repair(&winner)
13052 .await
13053 .unwrap_err()
13054 .is::<crate::repo_lock::ResourceBlocked>()
13055 );
13056 assert!(matches!(
13057 runner.state.recovery.as_ref().unwrap().repair,
13058 crate::recovery::Repair::Started { .. }
13059 ));
13060 assert!(leftover.exists());
13061 drop(external);
13062 runner.recovery_repair(&winner).await.unwrap();
13063 assert!(!leftover.exists());
13064 assert!(matches!(
13065 runner.state.recovery.as_ref().unwrap().repair,
13066 crate::recovery::Repair::Done { .. }
13067 ));
13068 std::fs::remove_file(&lock_path).unwrap();
13070 std::fs::create_dir(&lock_path).unwrap();
13071 std::fs::write(&leftover, "untrusted edits").unwrap();
13072 let (_, _, out) = runner
13073 .ask_fixer_among(
13074 attempts,
13075 "graft",
13076 None,
13077 Some((&winner.worktree, &head)),
13078 build,
13079 )
13080 .await;
13081 assert!(matches!(out, AgentOutcome::Failed(_)));
13082 assert!(leftover.exists());
13083 assert!(!next.exists());
13084 assert_eq!(
13085 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13086 head
13087 );
13088 }
13089
13090 #[cfg(unix)]
13091 async fn registered_runner(root: &Path) -> Runner {
13092 ask_test_home();
13093 let repo = root.join("repo");
13094 std::fs::create_dir_all(&repo).unwrap();
13095 init_repo(&repo);
13096 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13097 let mut runner = runner_at(RunStatus::Prep);
13098 runner.state.repo = repo.clone();
13099 runner.state.config.coordination = crate::config::test_coordination();
13100 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13101 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13102 runner.state.config.graph.advise = false;
13103 runner.roles.implementers = vec![spec("implement")];
13104 runner.roles.judges = vec![spec("judge")];
13105 runner.prep().await.unwrap();
13106 runner
13107 }
13108
13109 #[tokio::test]
13110 #[cfg(unix)]
13111 async fn parked_implementer_persists_completed_work() {
13112 for registered in [false, true] {
13113 let tmp = tempfile::tempdir().unwrap();
13114 let mut runner = registered_runner(tmp.path()).await;
13115 if !registered {
13116 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13117 }
13118 let marker = tmp.path().join("started");
13119 let release = tmp.path().join("release");
13120 let mut implementer = spec("implement");
13121 implementer.command = vec![
13122 "sh".into(),
13123 "-c".into(),
13124 format!(
13125 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13126 marker.display(),
13127 release.display()
13128 ),
13129 ];
13130 runner.roles.implementers = vec![implementer.clone()];
13131 runner.state.config.agents = vec![implementer];
13132 let pause = runner.pause.clone();
13133 let (result, ()) = tokio::join!(runner.implement(), async {
13134 tokio::time::timeout(Duration::from_secs(10), async {
13135 while !marker.exists() {
13136 tokio::time::sleep(Duration::from_millis(10)).await;
13137 }
13138 })
13139 .await
13140 .unwrap();
13141 pause.park();
13142 std::fs::write(&release, "go").unwrap();
13143 });
13144 result.unwrap();
13145 let candidate = &runner.state.candidates[0];
13146 assert!(candidate.commits > 0, "{candidate:?}");
13147 assert!(candidate.summary.contains("saved-result"));
13148 assert_eq!(candidate.files, 1);
13149 let before = candidate.commits;
13150 runner.implement().await.unwrap();
13151 assert_eq!(runner.state.candidates[0].commits, before);
13152 }
13153 }
13154
13155 #[tokio::test]
13156 #[cfg(unix)]
13157 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13158 let tmp = tempfile::tempdir().unwrap();
13159 let mut runner = registered_runner(tmp.path()).await;
13160 let winner = runner.state.candidates[0].clone();
13161 let cache = tmp.path().join("cache");
13162 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13163 runner.state.config.graph.timeout_resource_wait = 0;
13164 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13165 let crate::cache::AcquireOutcome::Acquired(held) =
13166 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13167 else {
13168 panic!("cache must be free")
13169 };
13170 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13171 from_run: "earlier".into(),
13172 generation: 1,
13173 mode: crate::recovery::Mode::VerifyRepair,
13174 reasons: Vec::new(),
13175 reviewed_head: None,
13176 rounds: Vec::new(),
13177 open_findings: Vec::new(),
13178 evidence: None,
13179 criteria_inherited: true,
13180 repair: crate::recovery::Repair::NotStarted,
13181 });
13182 runner.recovery_repair(&winner).await.unwrap();
13183 assert!(matches!(
13184 runner.state.recovery.as_ref().unwrap().repair,
13185 crate::recovery::Repair::WaitingForRestore { .. }
13186 ));
13187 drop(held);
13188 runner.state.resource_block = None;
13189 runner.state.config.graph.timeout_resource_wait = 1;
13190 runner.recovery_repair(&winner).await.unwrap();
13191 assert!(matches!(
13192 runner.state.recovery.as_ref().unwrap().repair,
13193 crate::recovery::Repair::Done { .. }
13194 ));
13195 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13196 }
13197
13198 #[tokio::test]
13199 #[cfg(unix)]
13200 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13201 use std::os::fd::AsRawFd as _;
13202 for graft in [false, true] {
13203 let tmp = tempfile::tempdir().unwrap();
13204 let mut runner = registered_runner(tmp.path()).await;
13205 runner.state.config.graph.timeout_resource_wait = 1;
13206 let winner = runner.state.candidates[0].clone();
13207 let marker = tmp.path().join("fixer-started");
13208 let mut fixer = spec("fixer");
13209 fixer.command = vec![
13210 "sh".into(),
13211 "-c".into(),
13212 format!("echo started >> '{}'; echo done", marker.display()),
13213 ];
13214 runner.roles.fixer = Some(vec![fixer]);
13215 if graft {
13216 let mut loser = winner.clone();
13217 loser.label = 'B';
13218 loser.index = 1;
13219 runner.state.candidates.push(loser);
13220 runner.state.config.graph.graft_runner_up = true;
13221 runner.state.tally = Some(Tally {
13222 first_choice: BTreeMap::new(),
13223 borda: BTreeMap::new(),
13224 winner: winner.label,
13225 rankings: 1,
13226 unanimous_initial: true,
13227 deliberated: false,
13228 changed_votes: 0,
13229 unanimous_final: true,
13230 tie_break: None,
13231 judges: 1,
13232 present: 1,
13233 quorum: 1,
13234 met_quorum: true,
13235 uncontested: None,
13236 });
13237 runner.state.judgements.push(crate::run::Judgement {
13238 judge: 1,
13239 seat: "judge-1".into(),
13240 agent: "judge".into(),
13241 ranking: vec![winner.label, 'B'],
13242 reasons: BTreeMap::new(),
13243 confidence: None,
13244 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13245 order: vec![0, 1],
13246 failed: None,
13247 duration_ms: 0,
13248 });
13249 } else {
13250 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13251 from_run: "earlier".into(),
13252 generation: 1,
13253 mode: crate::recovery::Mode::VerifyRepair,
13254 reasons: Vec::new(),
13255 reviewed_head: None,
13256 rounds: Vec::new(),
13257 open_findings: Vec::new(),
13258 evidence: None,
13259 criteria_inherited: true,
13260 repair: crate::recovery::Repair::NotStarted,
13261 });
13262 }
13263 let reader = crate::repo_lock::SeatLease::for_checkout(
13264 &winner.worktree,
13265 false,
13266 crate::config::test_coordination().as_ref(),
13267 )
13268 .await
13269 .unwrap();
13270 let pause = runner.pause.clone();
13271 let (result, ()) = tokio::join!(
13272 async {
13273 if graft {
13274 runner.graft().await
13275 } else {
13276 runner.recovery_repair(&winner).await
13277 }
13278 },
13279 async {
13280 tokio::time::sleep(Duration::from_millis(100)).await;
13281 pause.park();
13282 }
13283 );
13284 result.unwrap();
13285 assert!(!marker.exists(), "a blocked seat must not spawn");
13286 runner.state = RunState::load(&runner.state.id).unwrap();
13288 if graft {
13289 assert_eq!(
13290 runner.state.graft.as_ref().unwrap().outcome,
13291 GraftOutcome::WaitingForRestore
13292 );
13293 } else {
13294 assert!(matches!(
13295 runner.state.recovery.as_ref().unwrap().repair,
13296 crate::recovery::Repair::WaitingForRestore { .. }
13297 ));
13298 }
13299 drop(reader);
13300 runner.pause = Pause::new();
13301 runner.state.resource_block = None;
13302 let external =
13304 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13305 assert_eq!(
13306 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13307 0
13308 );
13309 let leftover = winner.worktree.join("leftover");
13310 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13311 let result = if graft {
13312 runner.graft().await
13313 } else {
13314 runner.recovery_repair(&winner).await
13315 };
13316 assert!(
13317 result
13318 .unwrap_err()
13319 .is::<crate::repo_lock::ResourceBlocked>()
13320 );
13321 assert!(leftover.exists());
13322 assert!(!marker.exists());
13323 drop(external);
13324 runner.state.resource_block = None;
13325 let result = if graft {
13326 runner.graft().await
13327 } else {
13328 runner.recovery_repair(&winner).await
13329 };
13330 result.unwrap();
13331 assert!(!leftover.exists(), "restore must precede retry");
13332 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13333 if graft {
13334 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13335 } else {
13336 assert!(matches!(
13337 runner.state.recovery.as_ref().unwrap().repair,
13338 crate::recovery::Repair::Done { .. }
13339 ));
13340 }
13341 }
13342 }
13343
13344 #[tokio::test]
13345 #[cfg(unix)]
13346 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13347 use std::os::fd::AsRawFd as _;
13348 let tmp = tempfile::tempdir().unwrap();
13349 let mut runner = registered_runner(tmp.path()).await;
13350 runner.state.config.graph.timeout_resource_wait = 1;
13351 let winner = runner.state.candidates[0].clone();
13352 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13353 runner.state.tally = Some(Tally {
13354 first_choice: BTreeMap::new(),
13355 borda: BTreeMap::new(),
13356 winner: winner.label,
13357 rankings: 0,
13358 unanimous_initial: false,
13359 deliberated: false,
13360 changed_votes: 0,
13361 unanimous_final: false,
13362 tie_break: None,
13363 judges: 0,
13364 present: 0,
13365 quorum: 0,
13366 met_quorum: true,
13367 uncontested: Some("fixture".into()),
13368 });
13369 runner.state.graft = Some(GraftRecord {
13370 pre: pre.clone(),
13371 post: None,
13372 lines: None,
13373 limit: None,
13374 agent: None,
13375 outcome: GraftOutcome::Running,
13376 requested: Vec::new(),
13377 adoption: None,
13378 adoption_error: None,
13379 });
13380 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13381 git::commit_all(&winner.worktree, "unfinished graft")
13382 .await
13383 .unwrap();
13384 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13385 let leftover = winner.worktree.join("leftover");
13386 std::fs::write(&leftover, "untrusted edits").unwrap();
13387 let path = runner.state.repo.join(".git/updater.lock");
13388 let external = std::fs::File::create(&path).unwrap();
13389 assert_eq!(
13390 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13391 0
13392 );
13393 assert!(
13394 runner
13395 .graft()
13396 .await
13397 .unwrap_err()
13398 .is::<crate::repo_lock::ResourceBlocked>()
13399 );
13400 assert!(matches!(
13401 runner.state.graft.as_ref().unwrap().outcome,
13402 GraftOutcome::Running
13403 ));
13404 assert_eq!(
13405 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13406 unfinished
13407 );
13408 assert!(leftover.exists());
13409 drop(external);
13410 std::fs::remove_file(&path).unwrap();
13411 std::fs::create_dir(&path).unwrap();
13412 assert!(runner.graft().await.is_err());
13413 assert!(matches!(
13414 runner.state.graft.as_ref().unwrap().outcome,
13415 GraftOutcome::Running
13416 ));
13417 assert_eq!(
13418 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13419 unfinished
13420 );
13421 assert!(leftover.exists());
13422 std::fs::remove_dir(&path).unwrap();
13423 runner.graft().await.unwrap();
13424 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13425 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13426 assert!(!leftover.exists());
13427 }
13428
13429 #[tokio::test]
13430 #[cfg(unix)]
13431 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13432 use std::os::fd::AsRawFd as _;
13433 let tmp = tempfile::tempdir().unwrap();
13434 let mut runner = registered_runner(tmp.path()).await;
13435 runner.state.prep_complete = Some(false);
13436 runner.state.config.graph.timeout_resource_wait = 1;
13437 runner.state.save().unwrap();
13438 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13439 assert_eq!(
13440 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13441 0
13442 );
13443 runner.execute().await.unwrap();
13444 let disk = RunState::load(&runner.state.id).unwrap();
13445 assert!(disk.driver_exited);
13446 assert_eq!(disk.driver_pid, None);
13447 assert_eq!(runner.state.status, RunStatus::Blocked);
13448 assert_eq!(
13449 runner.state.resource_block.as_ref().unwrap().kind,
13450 "repository coordination"
13451 );
13452 assert!(runner.state.prep_failure.is_none());
13453 assert!(runner.state.seats.is_empty());
13454 runner.pause.park();
13455 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13458 .await
13459 .unwrap()
13460 .unwrap();
13461 assert!(runner.state.parked);
13462 assert!(runner.state.prep_failure.is_none());
13463 let disk = RunState::load(&runner.state.id).unwrap();
13464 assert!(disk.driver_exited);
13465 assert_eq!(disk.driver_pid, None);
13466 }
13467
13468 #[tokio::test]
13469 #[cfg(unix)]
13470 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13471 use std::os::fd::AsRawFd as _;
13472 let tmp = tempfile::tempdir().unwrap();
13473 let mut runner = registered_runner(tmp.path()).await;
13474 runner.state.prep_complete = Some(false);
13475 runner.state.config.graph.timeout_resource_wait = 30;
13476 runner.state.save().unwrap();
13477 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13478 assert_eq!(
13479 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13480 0
13481 );
13482 assert!(
13483 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13484 .await
13485 .is_err()
13486 );
13487 let disk = RunState::load(&runner.state.id).unwrap();
13488 assert!(disk.driver_exited);
13489 assert_eq!(disk.driver_pid, None);
13490 assert!(disk.status.resumable());
13491 assert!(runner.state.driver_exited);
13492 assert_eq!(runner.state.driver_pid, None);
13493 runner.state.save().unwrap();
13494 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13495 }
13496
13497 #[tokio::test]
13498 #[cfg(unix)]
13499 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13500 let tmp = tempfile::tempdir().unwrap();
13501 let mut runner = registered_runner(tmp.path()).await;
13502 let winner = runner.state.candidates[0].clone();
13503 let cache = tmp.path().join("cache");
13504 let started = tmp.path().join("pre-gate-started");
13505 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13506 runner.state.config.verify.gate =
13507 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13508 let home = crate::run::home();
13509 let owner =
13510 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13511 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13512 crate::cache::AcquireOutcome::Acquired(g) => g,
13513 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13514 };
13515 let mut gate = Box::pin(runner.run_gate(&winner));
13516 assert!(
13517 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13518 .await
13519 .is_err()
13520 );
13521 assert!(!started.exists(), "pre-gate must wait for the cache too");
13522 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13526 let editor = crate::repo_lock::SeatLease::checkout_with(
13527 &winner.worktree,
13528 true,
13529 crate::config::test_coordination().as_ref(),
13530 &wait,
13531 )
13532 .await
13533 .unwrap();
13534 drop(editor);
13535 guard.release();
13536 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13537 .await
13538 .unwrap()
13539 .unwrap();
13540 drop(gate);
13541 assert!(outcomes.iter().all(CommandOutcome::ok));
13542 assert!(started.exists());
13543 }
13544
13545 #[tokio::test]
13546 #[cfg(unix)]
13547 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13548 use std::os::fd::AsRawFd as _;
13549
13550 ask_test_home();
13551 let tmp = tempfile::tempdir().unwrap();
13552 let repo = tmp.path().join("repo");
13553 std::fs::create_dir_all(&repo).unwrap();
13554 init_repo(&repo);
13555 let common = repo.join(".git");
13556 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13557 let mut runner = runner_at(RunStatus::Prep);
13558 runner.state.repo = repo.clone();
13559 runner.state.config.coordination = crate::config::test_coordination();
13560 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13561 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13562 runner.state.config.graph.advise = false;
13563 runner.roles.implementers = vec![spec("implement")];
13564 runner.roles.judges = vec![spec("judge")];
13565 runner.prep().await.unwrap();
13566 let winner = runner.state.candidates[0].clone();
13567 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13570 let completed = tmp.path().join("seat-completed");
13571 runner.roles.implementers[0].command = vec![
13572 "sh".into(),
13573 "-c".into(),
13574 format!("touch '{}'; echo completed", completed.display()),
13575 ];
13576 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13577 assert_eq!(
13578 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13579 0
13580 );
13581 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13582 let mut implement = Box::pin(runner.implement());
13583 tokio::select! {
13584 result = &mut implement => panic!("rescue must wait: {result:?}"),
13585 result = tokio::time::timeout(Duration::from_secs(10), async {
13586 while !completed.exists() {
13587 tokio::time::sleep(Duration::from_millis(20)).await;
13588 }
13589 }) => result.unwrap(),
13590 }
13591 assert!(
13592 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13593 .await
13594 .is_err()
13595 );
13596 assert_eq!(
13597 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13598 before
13599 );
13600 assert_eq!(
13601 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13602 0
13603 );
13604 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13605 .await
13606 .unwrap()
13607 .unwrap();
13608 drop(implement);
13609 assert_ne!(
13610 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13611 before
13612 );
13613 assert!(git::is_clean(&winner.worktree).await.unwrap());
13614
13615 let started = tmp.path().join("pre-gate-started");
13616 let proceed = tmp.path().join("pre-gate-proceed");
13617 let checked = tmp.path().join("gate-ran");
13618 runner.state.config.verify.pre_gate = vec![format!(
13619 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13620 started.display(),
13621 proceed.display()
13622 )];
13623 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13624 assert_eq!(
13625 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13626 0
13627 );
13628 let mut gate = Box::pin(runner.run_gate(&winner));
13629 assert!(
13630 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13631 .await
13632 .is_err()
13633 );
13634 assert!(!started.exists());
13635 assert!(!checked.exists());
13636 assert!(git::is_clean(&winner.worktree).await.unwrap());
13637 assert_eq!(
13638 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13639 0
13640 );
13641 tokio::select! {
13642 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13643 result = tokio::time::timeout(Duration::from_secs(10), async {
13644 while !started.exists() {
13645 tokio::time::sleep(Duration::from_millis(20)).await;
13646 }
13647 }) => result.unwrap(),
13648 }
13649 assert_eq!(
13650 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13651 -1,
13652 "mechanical edits must hold the real lock"
13653 );
13654 std::fs::write(proceed, "ready").unwrap();
13655 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13656 .await
13657 .unwrap()
13658 .unwrap();
13659 drop(gate);
13660 assert!(outcomes.iter().all(CommandOutcome::ok));
13661 assert!(checked.exists());
13662 assert!(runner.state.pre_gate_commit.is_some());
13663 assert!(git::is_clean(&winner.worktree).await.unwrap());
13664 assert_eq!(
13665 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13666 0,
13667 "gate completion must release coordination"
13668 );
13669 drop(outsider);
13670 std::fs::remove_file(common.join("updater.lock")).unwrap();
13671 std::fs::create_dir(common.join("updater.lock")).unwrap();
13672 std::fs::remove_file(&started).unwrap();
13673 std::fs::remove_file(&checked).unwrap();
13674 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13675 let error = runner.run_gate(&winner).await.unwrap_err();
13676 assert!(
13677 error
13678 .to_string()
13679 .contains("acquire mandatory repository lock")
13680 );
13681 assert!(
13682 !started.exists(),
13683 "failed acquisition must prevent pre-gate"
13684 );
13685 assert!(!checked.exists(), "failed acquisition must prevent gate");
13686 assert!(git::is_clean(&winner.worktree).await.unwrap());
13687 assert_eq!(
13688 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13689 head
13690 );
13691 }
13692
13693 #[tokio::test]
13694 #[cfg(unix)]
13695 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13696 use std::os::fd::AsRawFd as _;
13697 use std::time::Duration;
13698
13699 ask_test_home();
13700 let tmp = tempfile::tempdir().unwrap();
13701 let repo = tmp.path().join("repo");
13702 std::fs::create_dir_all(&repo).unwrap();
13703 init_repo(&repo);
13704 let common = repo.join(".git");
13705 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13706 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13707 assert_eq!(
13708 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13709 0
13710 );
13711 let mut runner = runner_at(RunStatus::Prep);
13712 runner.state.repo = repo.clone();
13713 runner.state.config.coordination = crate::config::test_coordination();
13714 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13715 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13716 runner.state.config.graph.advise = true;
13717 runner.state.config.graph.advisors = 1;
13718 runner.roles.implementers = vec![spec("implement")];
13719 runner.roles.judges = vec![spec("judge")];
13720 let root = runner.state.worktree_root();
13721 let started = tmp.path().join("setup-started");
13722 let proceed = tmp.path().join("setup-proceed");
13723 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13724 run: Some(format!(
13725 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13726 started.display(),
13727 proceed.display()
13728 )),
13729 ..Default::default()
13730 }];
13731 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13732 let config = git::git(&repo, &["config", "--local", "--list"])
13733 .await
13734 .unwrap();
13735 let mut prep = Box::pin(runner.prep());
13736 assert!(
13737 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13738 .await
13739 .is_err()
13740 );
13741 assert!(
13742 !root.exists(),
13743 "contention must prevent even checkout creation"
13744 );
13745 assert!(!started.exists(), "contention must prevent setup");
13746 assert_eq!(
13747 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13748 branches
13749 );
13750 assert_eq!(
13751 git::git(&repo, &["config", "--local", "--list"])
13752 .await
13753 .unwrap(),
13754 config
13755 );
13756 drop(lock);
13757 tokio::select! {
13758 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13759 result = tokio::time::timeout(Duration::from_secs(10), async {
13760 while !started.exists() {
13761 tokio::time::sleep(Duration::from_millis(20)).await;
13762 }
13763 }) => result.unwrap(),
13764 }
13765 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13766 assert_eq!(
13767 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13768 -1
13769 );
13770 assert!(
13771 tokio::time::timeout(
13772 Duration::from_millis(100),
13773 crate::repo_lock::SeatLease::for_checkout(
13774 &repo,
13775 true,
13776 crate::config::test_coordination().as_ref()
13777 )
13778 )
13779 .await
13780 .is_err(),
13781 "preparation must also exclude local editors"
13782 );
13783 std::fs::write(proceed, "ready").unwrap();
13784 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13785 .await
13786 .unwrap()
13787 .unwrap();
13788 drop(prep);
13789 assert_eq!(runner.state.prep_complete, Some(true));
13790 assert!(root.join("judge-1").exists());
13791 assert!(root.join("advisor-1").exists());
13792 assert_eq!(
13793 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13794 0
13795 );
13796 }
13797
13798 #[tokio::test]
13799 #[cfg(unix)]
13800 async fn preparation_lock_failure_prevents_repository_changes() {
13801 ask_test_home();
13802 let tmp = tempfile::tempdir().unwrap();
13803 let repo = tmp.path().join("repo");
13804 std::fs::create_dir_all(&repo).unwrap();
13805 init_repo(&repo);
13806 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13807 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13810 let mut runner = runner_at(RunStatus::Prep);
13811 runner.state.repo = repo.clone();
13812 runner.state.config.coordination = crate::config::test_coordination();
13813 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13814 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13815 runner.roles.implementers = vec![spec("implement")];
13816 let error = runner.prep().await.unwrap_err();
13817 assert!(
13818 error
13819 .to_string()
13820 .contains("acquire mandatory repository lock")
13821 );
13822 assert!(!runner.state.worktree_root().exists());
13823 assert!(runner.state.candidates.is_empty());
13824 assert!(!repo.join(".git/config.worktree").exists());
13825 assert_eq!(
13826 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13827 "* main"
13828 );
13829 }
13830
13831 #[tokio::test]
13832 async fn preparation_records_creation_intent_before_git_can_fail() {
13833 ask_test_home();
13834 let tmp = tempfile::tempdir().unwrap();
13835 let repo = tmp.path().join("repo");
13836 std::fs::create_dir_all(&repo).unwrap();
13837 init_repo(&repo);
13838 let mut runner = runner_at(RunStatus::Prep);
13839 runner.state.repo = repo.clone();
13840 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13841 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13842 runner.state.config.blind.commit_msg_hook = false;
13843 runner.roles.implementers = vec![spec("one")];
13844 let label = blind::assign_labels(1, runner.state.seed)[0];
13845 let branch = runner.state.branch_for(label);
13846 git::git(&repo, &["branch", &branch]).await.unwrap();
13849 assert!(runner.prep().await.is_err());
13850 let saved = RunState::load(&runner.state.id).unwrap();
13851 assert!(saved.candidates.is_empty());
13852 let event = saved
13853 .events
13854 .iter()
13855 .find(|e| e.node == "prep-intent")
13856 .unwrap();
13857 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13858 assert_eq!(intent["branch"], branch);
13859 assert_eq!(intent["start"], runner.state.base_commit);
13860 assert_eq!(
13861 intent["worktree"],
13862 runner
13863 .state
13864 .worktree_root()
13865 .join(format!("cand-{label}"))
13866 .to_string_lossy()
13867 .as_ref()
13868 );
13869 assert_eq!(intent["picks"], serde_json::json!([]));
13870 }
13871
13872 #[tokio::test]
13873 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13874 ask_test_home();
13875 let tmp = tempfile::tempdir().unwrap();
13876 let repo = tmp.path().join("repo");
13877 std::fs::create_dir_all(&repo).unwrap();
13878 init_repo(&repo);
13879 let mut runner = runner_at(RunStatus::Prep);
13880 runner.state.repo = repo.clone();
13881 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13882 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13883 runner.state.sources.pick_commits.push(refs::Input {
13884 token: "missing".into(),
13885 sha: "0".repeat(40),
13886 provenance: refs::Provenance::ExplicitCli,
13887 });
13888 assert!(runner.prep().await.is_err());
13889 assert!(runner.state.candidates.is_empty());
13890 assert!(!runner.state.worktree_root().exists());
13891 }
13892
13893 #[tokio::test]
13894 async fn prose_and_legacy_inference_never_import_into_candidates() {
13895 ask_test_home();
13896 let tmp = tempfile::tempdir().unwrap();
13897 let repo = tmp.path().join("repo");
13898 std::fs::create_dir_all(&repo).unwrap();
13899 init_repo(&repo);
13900 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13901 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13902 .await
13903 .unwrap();
13904 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13905 git::git(&repo, &["add", "."]).await.unwrap();
13906 git::git(&repo, &["commit", "-m", "historical implementation"])
13907 .await
13908 .unwrap();
13909 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13910 git::git(&repo, &["checkout", "main"]).await.unwrap();
13911 for instruction in [
13912 format!("Example: magi/history/A at {source}"),
13913 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13914 format!("Never import magi/history/A or {source}"),
13915 format!("Conversation answer: I mentioned {source} only as background"),
13916 ] {
13917 let mut runner = runner_at(RunStatus::Prep);
13918 runner.state.repo = repo.clone();
13919 runner.state.base_commit = base.clone();
13920 runner.state.instruction = instruction.clone();
13921 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13922 runner.state.config.blind.commit_msg_hook = false;
13923 runner.state.seeds = vec![refs::Seed {
13924 token: "magi/history/A".into(),
13925 sha: source.clone(),
13926 kind: refs::SeedKind::Unmerged,
13927 branch: true,
13928 detail: String::new(),
13929 }];
13930 runner.roles.implementers = vec![spec("one")];
13931 assert_eq!(seeded_instruction(&runner.state), instruction);
13932 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13934 legacy.as_object_mut().unwrap().remove("sources");
13935 runner.state = serde_json::from_value(legacy).unwrap();
13936 assert_eq!(runner.state.sources, refs::Sources::default());
13937 runner.state.save().unwrap();
13938 runner.state = RunState::load(&runner.state.id).unwrap();
13939 runner.prep().await.unwrap();
13940 for candidate in &runner.state.candidates {
13941 assert_eq!(
13942 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13943 base
13944 );
13945 assert!(!candidate.worktree.join("unwanted").exists());
13946 }
13947 }
13948 }
13949
13950 #[tokio::test]
13951 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13952 ask_test_home();
13953 let tmp = tempfile::tempdir().unwrap();
13954 let repo = tmp.path().join("repo");
13955 std::fs::create_dir_all(&repo).unwrap();
13956 init_repo(&repo);
13957 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13958 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13959 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13960 git::git(&repo, &["add", "."]).await.unwrap();
13961 git::git(&repo, &["commit", "-m", "required seed"])
13962 .await
13963 .unwrap();
13964 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13965 git::git(&repo, &["checkout", "main"]).await.unwrap();
13966 let mut runner = runner_at(RunStatus::Prep);
13967 runner.state.repo = repo.clone();
13968 runner.state.base_commit = base.clone();
13969 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13970 runner.state.config.graph.advise = false;
13971 runner.state.config.blind.commit_msg_hook = false;
13972 runner.state.sources =
13973 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13974 .await
13975 .unwrap();
13976 runner.roles.implementers = vec![spec("one"), spec("two")];
13977 runner.roles.judges = vec![spec("judge")];
13978 let label = blind::assign_labels(2, runner.state.seed)[0];
13979 let branch = runner.state.branch_for(label);
13980 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13981 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13982 .await
13983 .unwrap();
13984 runner.state.candidates.push(Candidate {
13985 index: 0,
13986 label,
13987 agent: "one".into(),
13988 branch,
13989 worktree,
13990 summary: String::new(),
13991 stat: String::new(),
13992 files: 0,
13993 commits: 0,
13994 empty: false,
13995 failed: None,
13996 verified_noop: None,
13997 duration_ms: 0,
13998 folded: false,
13999 });
14000 runner.state.save().unwrap();
14001 runner.state = RunState::load(&runner.state.id).unwrap();
14003 assert_eq!(runner.state.prep_complete, Some(false));
14004 runner.prep().await.unwrap();
14005 assert_eq!(runner.state.prep_complete, Some(true));
14006 assert_eq!(runner.state.candidates.len(), 2);
14007 for candidate in &runner.state.candidates {
14008 assert_eq!(
14009 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14010 git::tree_of(&repo, &seed).await.unwrap()
14011 );
14012 assert_eq!(
14013 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14014 "required commit"
14015 );
14016 }
14017 assert!(runner.state.worktree_root().join("judge-1").exists());
14018 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14019 runner.prep().await.unwrap();
14020 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14021
14022 runner.state.prep_complete = Some(false);
14023 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14024 run: Some(format!(
14025 "test -f '{}'",
14026 tmp.path().join("dependency-ready").display()
14027 )),
14028 ..Default::default()
14029 }];
14030 assert!(runner.prep().await.is_err());
14031 assert!(runner.state.prep_failure.is_some());
14032 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14033 runner.state = RunState::load(&runner.state.id).unwrap();
14034 assert!(
14035 runner.prep().await.is_err(),
14036 "correcting the cause alone must not auto-retry"
14037 );
14038 let failed_at = runner.state.events.last().unwrap().at;
14039 assert!(
14040 runner.retry_preparation(failed_at).is_err(),
14041 "old approvals cannot unlock the hold"
14042 );
14043 let mut task = crate::queue::Task::new(
14044 "retry".into(),
14045 "retry setup".into(),
14046 repo.clone(),
14047 crate::queue::Source::Human,
14048 );
14049 task.start(runner.state.id.clone());
14050 task.hold_machine(Some("setup failed".into()));
14051 task.release_operator("cli", None);
14052 runner
14053 .retry_preparation(task.resumption.as_ref().unwrap().at)
14054 .unwrap();
14055 runner.prep().await.unwrap();
14056 assert!(runner.state.prep_failure.is_none());
14057 assert_eq!(runner.state.prep_complete, Some(true));
14058 assert_eq!(runner.state.candidates.len(), 2);
14059 assert!(runner.state.worktree_root().join("judge-1").exists());
14060 for candidate in &runner.state.candidates {
14061 assert_eq!(
14062 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14063 git::tree_of(&repo, &seed).await.unwrap()
14064 );
14065 }
14066 }
14067
14068 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14069 let mut heads = Vec::new();
14070 for candidate in candidates {
14071 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14072 }
14073 heads
14074 }
14075
14076 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14077 ask_test_home();
14078 let tmp = tempfile::tempdir().unwrap();
14079 let repo = tmp.path().join("repo");
14080 std::fs::create_dir(&repo).unwrap();
14081 init_repo(&repo);
14082 let mut runner = runner_at(RunStatus::Reviewing);
14083 runner.state.repo = repo.clone();
14084 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14085 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14086 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14087 let branch = format!("candidate-{label}");
14088 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14089 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14090 .await
14091 .unwrap();
14092 if label == 'B' {
14093 std::fs::write(worktree.join("change"), "winner").unwrap();
14094 git::rescue_commit(&worktree, "candidate change")
14095 .await
14096 .unwrap();
14097 }
14098 runner.state.candidates.push(Candidate {
14099 index,
14100 label,
14101 agent: "alpha".into(),
14102 branch,
14103 worktree,
14104 summary: "completed reply".into(),
14105 stat: String::new(),
14106 files: usize::from(label == 'B'),
14107 commits: usize::from(label == 'B'),
14108 empty: label == 'A',
14109 failed: None,
14110 verified_noop: None,
14111 duration_ms: 1,
14112 folded: false,
14113 });
14114 }
14115 runner.state.judge_skipped = true;
14116 runner.state.tally = Some(Tally {
14117 first_choice: BTreeMap::new(),
14118 borda: BTreeMap::new(),
14119 winner: 'B',
14120 rankings: 0,
14121 unanimous_initial: false,
14122 deliberated: false,
14123 changed_votes: 0,
14124 unanimous_final: false,
14125 tie_break: None,
14126 judges: 0,
14127 present: 0,
14128 quorum: 0,
14129 met_quorum: true,
14130 uncontested: Some("only B produced a change".into()),
14131 });
14132 runner.state.save().unwrap();
14133 (tmp, runner)
14134 }
14135
14136 #[tokio::test]
14137 #[cfg(unix)]
14138 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14139 for (solo, failed_checkpoint) in
14140 [(true, false), (false, false), (true, true), (false, true)]
14141 {
14142 let (_tmp, mut runner) = preservation_resume_fixture().await;
14143 if solo {
14144 runner.state.candidates.retain(|c| c.label == 'A');
14145 }
14146 runner.state.candidates[0].empty = false;
14147 runner.state.candidates[0].summary.clear();
14148 runner.state.tally = None;
14149 runner.state.judge_skipped = false;
14150 runner.state.status = RunStatus::Implementing;
14151 runner.state.config.disk.min_free_bytes = u64::MAX;
14152 let mut implementer = spec("alpha");
14153 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14154 runner.roles.implementers = vec![implementer];
14155 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14156 if failed_checkpoint {
14157 std::fs::create_dir_all(&patch).unwrap();
14158 }
14159 runner.state.save().unwrap();
14160 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14161 runner.state = RunState::load(&runner.state.id).unwrap();
14162 assert_eq!(runner.state.status, RunStatus::Blocked);
14163 assert!(runner.state.interrupted_implementations.contains(&'A'));
14164 assert_eq!(runner.state.candidates[0].commits, 0);
14165 assert!(!runner.state.candidates[0].empty);
14166 assert!(runner.state.candidates[0].failed.is_none());
14167 if failed_checkpoint {
14168 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14169 std::fs::remove_dir(patch).unwrap();
14170 }
14171 runner.state.resource_block = None;
14172 runner.state.config.disk.min_free_bytes = 0;
14173 runner.roles.implementers[0].command = vec![
14174 "sh".into(),
14175 "-c".into(),
14176 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14177 ];
14178 runner.implement().await.unwrap();
14179 assert!(runner.state.interrupted_implementations.is_empty());
14180 assert!(runner.state.candidate_preservation.is_empty());
14181 assert_eq!(runner.state.candidates[0].commits, 1);
14182 assert!(!runner.state.candidates[0].empty);
14183 assert!(
14184 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14185 .unwrap()
14186 .contains("retried empty interrupted seat")
14187 );
14188 if !solo {
14189 assert_eq!(runner.state.candidates[1].commits, 1);
14190 }
14191 }
14192 }
14193
14194 #[tokio::test]
14195 #[cfg(unix)]
14196 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14197 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14198 let (_tmp, mut runner) = preservation_resume_fixture().await;
14199 runner.state.candidates.retain(|c| c.label == 'A');
14200 runner.state.candidates[0].empty = false;
14201 runner.state.candidates[0].summary.clear();
14202 runner.state.tally = None;
14203 runner.state.judge_skipped = false;
14204 runner.state.status = RunStatus::Implementing;
14205 runner.state.config.disk.min_free_bytes = u64::MAX;
14208 let wt = runner.state.candidates[0].worktree.clone();
14209 std::fs::write(
14210 wt.join("interrupted-source"),
14211 "retain implementation work\n",
14212 )
14213 .unwrap();
14214 if committed {
14215 git::rescue_commit(&wt, "implementation checkpoint")
14216 .await
14217 .unwrap();
14218 }
14219 let lock = git::git(
14220 &wt,
14221 &[
14222 "rev-parse",
14223 "--path-format=absolute",
14224 "--git-path",
14225 "index.lock",
14226 ],
14227 )
14228 .await
14229 .unwrap();
14230 if block_rescue {
14231 std::fs::write(&lock, "fixture-owned lock").unwrap();
14232 }
14233 let mut implementer = spec("alpha");
14234 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14235 runner.roles.implementers = vec![implementer];
14236 runner.state.save().unwrap();
14237 let result = runner.implement().await;
14238 assert_eq!(result.is_err(), block_rescue);
14239 assert_eq!(
14240 runner.state.resource_block.as_ref().unwrap().kind,
14241 "disk pressure"
14242 );
14243 assert!(
14244 runner
14245 .state
14246 .events
14247 .iter()
14248 .any(|e| e.message.contains("agent execution interrupted"))
14249 );
14250 assert!(
14251 runner
14252 .state
14253 .dir()
14254 .join("artifacts/candidate-A-recovery/staged.patch")
14255 .exists()
14256 );
14257 if block_rescue {
14258 let saved = RunState::load(&runner.state.id).unwrap();
14259 assert!(saved.candidate_preservation.contains_key(&'A'));
14260 assert!(!saved.candidates[0].empty);
14261 assert_eq!(
14262 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14263 "retain implementation work\n"
14264 );
14265 std::fs::remove_file(lock).unwrap();
14266 runner.state = saved;
14267 } else {
14268 assert_eq!(runner.state.status, RunStatus::Blocked);
14269 assert_eq!(runner.state.candidates[0].commits, 1);
14270 }
14271 runner.roles.implementers.clear();
14274 runner.state.config.disk.min_free_bytes = 0;
14275 runner.state.resource_block = None;
14276 runner.implement().await.unwrap();
14277 assert!(runner.state.candidate_preservation.is_empty());
14278 assert_eq!(runner.state.candidates[0].commits, 1);
14279 assert!(!runner.state.candidates[0].empty);
14280 assert!(git::is_clean(&wt).await.unwrap());
14281 assert!(
14282 crate::run::read_artifact(&runner.state, "cand-A.patch")
14283 .unwrap()
14284 .contains("retain implementation work")
14285 );
14286 }
14287 }
14288
14289 #[tokio::test]
14290 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14291 let (_tmp, mut runner) = preservation_resume_fixture().await;
14292 runner.implement().await.unwrap();
14294 assert!(runner.state.candidate_preservation.is_empty());
14295 assert!(
14296 !runner
14297 .state
14298 .dir()
14299 .join("artifacts/candidate-A-recovery")
14300 .exists()
14301 );
14302 runner.fold_losers().await.unwrap();
14303 assert!(!runner.state.candidates[0].worktree.exists());
14304 for schema in [14, crate::run::SCHEMA] {
14305 runner.state.schema = schema;
14306 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14307 runner.state.save().unwrap();
14308 runner.state = RunState::load(&runner.state.id).unwrap();
14309 runner.implement().await.unwrap();
14310 assert!(runner.state.candidate_preservation.is_empty());
14311 assert!(runner.state.candidates[0].folded);
14312 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14313 }
14314 }
14315
14316 #[tokio::test]
14317 async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14318 let (_tmp, mut runner) = preservation_resume_fixture().await;
14319 runner.state.candidates.retain(|c| c.label == 'A');
14320 runner.state.tally = None;
14321 runner.state.judge_skipped = false;
14322 let candidate = &mut runner.state.candidates[0];
14323 candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14324 candidate.failed = Some("deployment validation unavailable".into());
14325 std::fs::write(
14326 candidate.worktree.join("repair"),
14327 "future launcher policy\n",
14328 )
14329 .unwrap();
14330 runner
14331 .state
14332 .candidate_preservation
14333 .insert('A', "awaiting persistence".into());
14334 runner.persist_candidates().await.unwrap();
14335 let candidate = &runner.state.candidates[0];
14336 assert!(!candidate.empty);
14337 assert_eq!(candidate.commits, 1);
14338 assert_eq!(candidate.files, 1);
14339 assert!(candidate.failed.is_none());
14340 assert!(candidate.summary.contains("EPERM"));
14341 assert!(
14342 crate::run::read_artifact(&runner.state, "cand-A.patch")
14343 .unwrap()
14344 .contains("future launcher policy")
14345 );
14346 assert!(runner.state.candidate_preservation.is_empty());
14347 }
14348
14349 #[tokio::test]
14350 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14351 let (_tmp, mut runner) = preservation_resume_fixture().await;
14352 let wt = runner.state.candidates[0].worktree.clone();
14353 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14354 runner.state.schema = 14;
14355 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14356 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14357 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14358 for _ in 0..2 {
14359 assert!(runner.implement().await.is_err());
14360 assert_eq!(runner.state.status, RunStatus::Blocked);
14361 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14362 assert!(
14363 runner.state.candidates[0].empty,
14364 "old verdict must remain unusable"
14365 );
14366 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14367 assert!(runner.state.judge_skipped);
14368 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14369 assert!(
14370 crate::run::read_artifact(&runner.state, "cand-A.patch")
14371 .unwrap()
14372 .contains("owned edits")
14373 );
14374 assert!(wt.exists());
14375 assert_eq!(
14376 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14377 .await
14378 .unwrap(),
14379 1
14380 );
14381 runner.state = RunState::load(&runner.state.id).unwrap();
14382 }
14383 }
14384
14385 #[tokio::test]
14386 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14387 let (_tmp, mut runner) = preservation_resume_fixture().await;
14388 let wt = runner.state.candidates[0].worktree.clone();
14389 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14390 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14391 let index_lock = PathBuf::from(
14392 git::git(
14393 &wt,
14394 &[
14395 "rev-parse",
14396 "--path-format=absolute",
14397 "--git-path",
14398 "index.lock",
14399 ],
14400 )
14401 .await
14402 .unwrap(),
14403 );
14404 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14405 runner.state.schema = 14;
14406 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14407 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14408 assert!(runner.implement().await.is_err());
14409 assert!(runner.after_implement().is_err());
14410 assert!(runner.fold_losers().await.is_err());
14411 assert_eq!(
14412 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14413 "failed seat edits"
14414 );
14415 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14416 runner.state = RunState::load(&runner.state.id).unwrap();
14417 std::fs::remove_file(index_lock).unwrap();
14418 assert!(runner.implement().await.is_err());
14420 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14421 assert!(runner.state.candidates[0].empty);
14422 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14423 assert!(
14424 crate::run::read_artifact(&runner.state, "cand-A.patch")
14425 .unwrap()
14426 .contains("failed seat edits")
14427 );
14428 assert!(wt.exists());
14429 }
14430
14431 #[tokio::test]
14432 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14433 let (_tmp, mut runner) = preservation_resume_fixture().await;
14434 let repo = runner.state.repo.clone();
14435 let common = git::git(
14436 &repo,
14437 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14438 )
14439 .await
14440 .unwrap();
14441 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14442 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14443 assert!(runner.fold_losers().await.is_err());
14444 assert!(!runner.state.candidates[0].worktree.exists());
14445 assert!(!runner.state.candidates[0].folded);
14446 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14447 runner.state = RunState::load(&runner.state.id).unwrap();
14448 std::fs::remove_file(branch_lock).unwrap();
14449 let judge = runner.state.worktree_root().join("judge-1");
14450 git::worktree_add_detached(&repo, &judge, "main")
14451 .await
14452 .unwrap();
14453 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14454 runner.roles.judges = vec![spec("judge")];
14455 assert!(runner.fold_losers().await.is_err());
14456 runner.state = RunState::load(&runner.state.id).unwrap();
14457 assert!(
14458 runner.state.candidates[0].folded,
14459 "candidate progress saved before judge failure"
14460 );
14461 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14462 assert!(judge.join("uncommitted").exists());
14463 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14464 runner.fold_losers().await.unwrap();
14465 assert!(!judge.exists());
14466 runner.fold_losers().await.unwrap();
14467 }
14468
14469 #[tokio::test]
14470 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14471 let (_tmp, mut runner) = preservation_resume_fixture().await;
14472 runner.state.candidates.retain(|c| c.label == 'B');
14473 runner.state.tally = None;
14474 runner.state.judge_skipped = false;
14475 runner.state.prep_complete = Some(true);
14476 runner.state.config.graph.advise = false;
14477 runner.state.config.graph.selected_implementer = Some("alpha".into());
14478 let wt = runner.state.candidates[0].worktree.clone();
14479 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14480 runner
14481 .state
14482 .candidate_preservation
14483 .insert('B', "awaiting persistence".into());
14484 let index_lock = git::git(
14485 &wt,
14486 &[
14487 "rev-parse",
14488 "--path-format=absolute",
14489 "--git-path",
14490 "index.lock",
14491 ],
14492 )
14493 .await
14494 .unwrap();
14495 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14496 runner.state.save().unwrap();
14497 assert!(runner.execute_graph().await.is_err());
14498 assert_eq!(runner.state.status, RunStatus::Blocked);
14499 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14500 std::fs::remove_file(index_lock).unwrap();
14501 for _ in 0..2 {
14504 runner.state = RunState::load(&runner.state.id).unwrap();
14505 runner.pause.park();
14506 runner.execute_graph().await.unwrap();
14507 assert_eq!(runner.state.status, RunStatus::Implementing);
14508 assert!(runner.state.candidate_preservation.is_empty());
14509 assert_eq!(runner.state.candidates[0].commits, 2);
14510 assert!(runner.state.tally.is_none());
14511 }
14512 runner.implement().await.unwrap();
14513 assert_eq!(runner.state.status, RunStatus::Judging);
14514 runner.state.status = RunStatus::Blocked;
14516 runner.state.candidates[0].failed = Some("implementation failed".into());
14517 runner.state.save().unwrap();
14518 let error = runner.execute_graph().await.unwrap_err();
14519 assert!(
14520 error
14521 .to_string()
14522 .contains("selected implementation is held")
14523 );
14524 }
14525
14526 #[tokio::test]
14527 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14528 let (_tmp, mut runner) = preservation_resume_fixture().await;
14529 let repo = runner.state.repo.clone();
14530 let candidate = runner.state.candidates[0].clone();
14531 let registered = git::worktree_holding(&repo, &candidate.branch)
14534 .await
14535 .unwrap()
14536 .unwrap();
14537 assert_eq!(
14538 registered.canonicalize().unwrap(),
14539 candidate.worktree.canonicalize().unwrap()
14540 );
14541 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14542 assert_eq!(
14543 git::worktree_holding(&repo, &candidate.branch)
14544 .await
14545 .unwrap(),
14546 Some(registered)
14547 );
14548 assert!(
14549 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14550 "the stale registration initially keeps the branch checked out"
14551 );
14552 runner.fold_losers().await.unwrap();
14553 assert!(runner.state.candidates[0].folded);
14554 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14555 assert_eq!(
14556 git::worktree_holding(&repo, &candidate.branch)
14557 .await
14558 .unwrap(),
14559 None
14560 );
14561 runner.state = RunState::load(&runner.state.id).unwrap();
14562 runner.fold_losers().await.unwrap();
14563 }
14564
14565 #[tokio::test]
14566 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14567 let (_tmp, mut runner) = preservation_resume_fixture().await;
14568 let repo = runner.state.repo.clone();
14569 let candidate = runner.state.candidates[0].clone();
14570 let winner = runner.state.candidates[1].clone();
14571 let registered = git::worktree_holding(&repo, &candidate.branch)
14572 .await
14573 .unwrap()
14574 .unwrap();
14575 assert_eq!(
14576 registered.canonicalize().unwrap(),
14577 candidate.worktree.canonicalize().unwrap()
14578 );
14579 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14580 assert_eq!(
14581 git::worktree_holding(&repo, &candidate.branch)
14582 .await
14583 .unwrap(),
14584 Some(registered)
14585 );
14586 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14587
14588 for _ in 0..2 {
14589 fold_run(&mut runner.state, false, &crate::run::home())
14590 .await
14591 .unwrap();
14592 runner.state = RunState::load(&runner.state.id).unwrap();
14593 assert!(runner.state.candidates[0].folded);
14594 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14595 assert_eq!(
14596 git::worktree_holding(&repo, &candidate.branch)
14597 .await
14598 .unwrap(),
14599 None
14600 );
14601 assert!(!runner.state.candidates[1].folded);
14602 assert!(winner.worktree.exists());
14603 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14604 }
14605 }
14606
14607 #[tokio::test]
14608 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14609 ask_test_home();
14610 let tmp = tempfile::tempdir().unwrap();
14611 let repo = tmp.path().join("repo");
14612 std::fs::create_dir(&repo).unwrap();
14613 init_repo(&repo);
14614 let mut runner = runner_at(RunStatus::Implementing);
14615 runner.state.repo = repo.clone();
14616 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14617 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14618 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14619 let branch = format!("candidate-{label}");
14620 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14621 git::worktree_add_branch(&repo, &wt, &branch, "main")
14622 .await
14623 .unwrap();
14624 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14625 git::git(&wt, &["add", "README.md"]).await.unwrap();
14626 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14627 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14628 runner.state.candidates.push(Candidate {
14629 index,
14630 label,
14631 agent: "alpha".to_owned(),
14632 branch,
14633 worktree: wt,
14634 summary: "completed implementation".to_owned(),
14635 stat: String::new(),
14636 files: 0,
14637 commits: 0,
14638 empty: false,
14639 failed: None,
14640 verified_noop: None,
14641 duration_ms: 42,
14642 folded: false,
14643 });
14644 runner
14645 .state
14646 .candidate_preservation
14647 .insert(label, "awaiting persistence".to_owned());
14648 }
14649 let wt = runner.state.candidates[0].worktree.clone();
14650 let index_lock = git::git(
14651 &wt,
14652 &[
14653 "rev-parse",
14654 "--path-format=absolute",
14655 "--git-path",
14656 "index.lock",
14657 ],
14658 )
14659 .await
14660 .unwrap();
14661 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14662 runner.state.save().unwrap();
14663 assert!(runner.persist_candidates().await.is_err());
14664 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14665 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14666 assert_eq!(
14667 runner.state.candidates[1].commits, 1,
14668 "other results are still collected"
14669 );
14670 assert!(
14671 !runner.state.candidates[0].empty,
14672 "failed rescue is not no change"
14673 );
14674 assert_eq!(
14675 runner.state.candidates[0].summary,
14676 "completed implementation"
14677 );
14678 assert!(runner.after_implement().is_err());
14679 let home = tmp.path().join("fold-home");
14680 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14681 assert!(wt.join("new-file").exists());
14682 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14683 assert!(recovery.join("staged.patch").exists());
14684 assert!(recovery.join("unstaged.patch").exists());
14685 assert_eq!(
14686 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14687 "untracked edits\n"
14688 );
14689 runner.state.begin_pass();
14690 runner.state.end_pass_parked(ParkCause::Upgrade {
14691 from: "1.0".into(),
14692 to: Some("1.1".into()),
14693 });
14694 runner.state.save().unwrap();
14695 runner.state = RunState::load(&runner.state.id).unwrap();
14696 assert!(matches!(
14697 runner.state.passes.last().unwrap().end,
14698 Some(crate::run::PassEnd::Parked {
14699 cause: ParkCause::Upgrade { .. }
14700 })
14701 ));
14702 std::fs::remove_file(index_lock).unwrap();
14703 runner.implement().await.unwrap();
14705 assert!(runner.state.candidate_preservation.is_empty());
14706 assert!(
14707 runner
14708 .state
14709 .candidates
14710 .iter()
14711 .all(|c| c.commits == 1 && !c.empty)
14712 );
14713 assert!(git::is_clean(&wt).await.unwrap());
14714 assert!(
14715 crate::run::read_artifact(&runner.state, "cand-A.patch")
14716 .unwrap()
14717 .contains("later tracked edits")
14718 );
14719 }
14720
14721 #[tokio::test]
14722 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14723 ask_test_home();
14724 let tmp = tempfile::tempdir().unwrap();
14725 let repo = tmp.path().join("repo");
14726 std::fs::create_dir(&repo).unwrap();
14727 init_repo(&repo);
14728 let wt = tmp.path().join("candidate");
14729 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14730 .await
14731 .unwrap();
14732 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14733 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14734 assert!(wt.join("uncommitted").exists());
14735 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14736 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14737 let mut runner = runner_at(RunStatus::Implementing);
14738 runner.state.repo = repo.clone();
14739 runner.state.base_commit = "missing-base".to_owned();
14740 runner.state.candidates.push(Candidate {
14741 index: 0,
14742 label: 'A',
14743 agent: "alpha".into(),
14744 branch: "candidate".into(),
14745 worktree: wt.clone(),
14746 summary: "completed".into(),
14747 stat: String::new(),
14748 files: 0,
14749 commits: 0,
14750 empty: false,
14751 failed: None,
14752 verified_noop: None,
14753 duration_ms: 1,
14754 folded: false,
14755 });
14756 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14757 assert!(
14758 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14759 .await
14760 .is_err()
14761 );
14762 assert!(!runner.state.candidates[0].folded);
14763 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14764 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14765 runner
14766 .state
14767 .candidate_preservation
14768 .insert('A', "pending".into());
14769 assert!(runner.persist_candidates().await.is_err());
14770 assert!(!runner.state.candidates[0].empty);
14771 assert!(!runner.state.candidates[0].folded);
14772 assert!(wt.exists());
14773 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14774 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14775 std::fs::create_dir_all(&patch_path).unwrap();
14776 assert!(
14777 runner.persist_candidates().await.is_err(),
14778 "artifact failure stays pending"
14779 );
14780 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14781 assert!(!runner.state.candidates[0].empty);
14782 std::fs::remove_dir(patch_path).unwrap();
14783 runner.persist_candidates().await.unwrap();
14784 assert!(
14785 runner.state.candidates[0].empty,
14786 "only successful inspection confirms no change"
14787 );
14788 }
14789
14790 fn init_repo(dir: &Path) {
14791 let run = |args: &[&str]| {
14792 let out = std::process::Command::new("git")
14793 .args(args)
14794 .current_dir(dir)
14795 .quiet()
14796 .output()
14797 .expect("spawn git");
14798 assert!(
14799 out.status.success(),
14800 "git {args:?} failed: {}",
14801 String::from_utf8_lossy(&out.stderr)
14802 );
14803 };
14804 run(&["init", "-b", "main"]);
14805 run(&["config", "user.name", "magi test"]);
14806 run(&["config", "user.email", "magi@example.com"]);
14807 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14808 run(&["add", "-A"]);
14809 run(&["commit", "-m", "init"]);
14810 }
14811
14812 fn ask_test_home() {
14820 crate::run::pin_test_home();
14821 }
14822
14823 fn runner_at(status: RunStatus) -> Runner {
14826 let mut state = RunState::new(
14827 PathBuf::from("/nonexistent/repo"),
14828 "main".to_owned(),
14829 "deadbeef".to_owned(),
14830 "task".to_owned(),
14831 Config::default(),
14832 );
14833 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14836 state.id = format!(
14837 "20000101-000000-{:04x}",
14838 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14839 );
14840 state.status = status;
14841 Runner {
14842 state,
14843 roles: ResolvedRoles {
14844 implementers: Vec::new(),
14845 judges: Vec::new(),
14846 reviewers: Vec::new(),
14847 fixer: None,
14848 conductor: conductor(),
14849 implementer_roster: Vec::new(),
14850 judge_roster: Vec::new(),
14851 reviewer_roster: Vec::new(),
14852 },
14853 sem: Arc::new(Semaphore::new(1)),
14854 pause: Pause::new(),
14855 interrupt: Pause::new(),
14856 }
14857 }
14858
14859 #[tokio::test]
14860 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14861 crate::run::pin_test_home();
14862 let mut runner = runner_at(RunStatus::Gating);
14863 runner.state.save().unwrap();
14864 let id = runner.state.id.clone();
14865 let passes = runner.state.passes.len();
14866 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14867 let mut claiming = Box::pin(runner.claim_driver());
14868 std::future::poll_fn(|cx| {
14869 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14870 std::task::Poll::Ready(())
14871 })
14872 .await;
14873 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14874 drop(owner);
14875 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14876 .await
14877 .unwrap()
14878 .unwrap();
14879 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14880 assert!(execution.runner.state.resource_block.is_none());
14881 assert_eq!(
14882 execution.runner.state.liveness(false),
14883 crate::run::Liveness::Live
14884 );
14885 drop(execution);
14886 assert!(RunState::load(&id).unwrap().driver_exited);
14887 }
14888
14889 #[tokio::test]
14890 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14891 crate::run::pin_test_home();
14892 let mut runner = runner_at(RunStatus::Gating);
14893 runner.state.save().unwrap();
14894 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14897 assert!(runner.claim_driver().await.is_err());
14898 assert!(runner.state.driver_exited);
14899 assert_eq!(runner.state.driver_pid, None);
14900 let disk = RunState::load(&runner.state.id).unwrap();
14901 assert!(disk.driver_exited);
14902 assert_eq!(disk.driver_pid, None);
14903 assert_eq!(disk.status, RunStatus::Gating);
14904 }
14905
14906 #[tokio::test]
14907 async fn an_early_graph_error_releases_its_driver() {
14908 crate::run::pin_test_home();
14909 let mut runner = runner_at(RunStatus::Blocked);
14910 runner.state.config.graph.selected_implementer = Some("selected".into());
14911 runner.state.save().unwrap();
14912 assert!(runner.execute().await.is_err());
14913 let disk = RunState::load(&runner.state.id).unwrap();
14914 assert!(disk.driver_exited);
14915 assert_eq!(disk.driver_pid, None);
14916 assert_eq!(disk.status, RunStatus::Blocked);
14917 }
14918
14919 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14920 crate::run::MergeRepair {
14921 question: "repair-question".into(),
14922 head: "old-head".into(),
14923 pr: "https://github.com/o/r/pull/42".into(),
14924 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14925 gate: vec![outcome(Some(0), false)],
14926 gate_head: Some("old-head".into()),
14927 gate_fixes: Vec::new(),
14928 pre_gate: Vec::new(),
14929 phase,
14930 failure: None,
14931 }
14932 }
14933
14934 #[tokio::test]
14935 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14936 crate::run::pin_test_home();
14937 let mut runner = runner_at(RunStatus::Landing);
14938 runner.state.merge_repairs.push(merge_repair_request(
14939 crate::run::MergeRepairPhase::Requested,
14940 ));
14941 runner.state.save().unwrap();
14942 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14943 assert!(
14944 runner
14945 .execute()
14946 .await
14947 .unwrap_err()
14948 .to_string()
14949 .contains("already running")
14950 );
14951 assert_eq!(
14952 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14953 crate::run::MergeRepairPhase::Requested
14954 );
14955 drop(claim);
14956 let mut disk = runner.state.clone();
14957 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14958 disk.save().unwrap();
14959 assert!(
14960 runner
14961 .execute()
14962 .await
14963 .unwrap_err()
14964 .to_string()
14965 .contains("decision changed")
14966 );
14967 assert_eq!(
14968 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14969 crate::run::MergeRepairPhase::Complete
14970 );
14971 }
14972
14973 #[test]
14974 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14975 crate::run::pin_test_home();
14976 let mut runner = runner_at(RunStatus::Landing);
14977 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14978 runner.state.reviews = request.reviews.clone();
14979 runner.state.gate = request.gate.clone();
14980 runner.state.gate_ran = true;
14981 runner.state.land_approval = Some(crate::run::LandApproval {
14982 question: "old".into(),
14983 head: "old-head".into(),
14984 });
14985 runner.state.merge_repairs.push(request);
14986 runner.begin_merge_repair_review(0).unwrap();
14987 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14988 assert!(!runner.state.gate_status().ok());
14989 assert!(runner.state.land_approval.is_none());
14990 let disk = RunState::load(&runner.state.id).unwrap();
14991 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14992 assert!(disk.merge_repairs[0].gate[0].ok());
14993 assert_eq!(
14994 disk.merge_repairs[0].phase,
14995 crate::run::MergeRepairPhase::Reviewing
14996 );
14997 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14998 assert!(disk.reviews.is_empty());
14999 }
15000
15001 #[tokio::test]
15002 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
15003 crate::run::pin_test_home();
15004 for phase in [
15005 crate::run::MergeRepairPhase::Fixing,
15006 crate::run::MergeRepairPhase::Requested,
15007 ] {
15008 let mut runner = runner_at(RunStatus::Landing);
15009 runner.state.config.graph.land_rounds = 0;
15010 runner.state.merge_repairs.push(merge_repair_request(phase));
15011 runner.state.save().unwrap();
15012 assert!(!runner.repair_merge_review().await.unwrap());
15013 assert_eq!(runner.state.status, RunStatus::Blocked);
15014 assert_eq!(
15015 runner.state.merge_repairs[0].phase,
15016 crate::run::MergeRepairPhase::Failed
15017 );
15018 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
15019 assert!(
15020 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
15021 "interrupted"
15022 } else {
15023 "budget exhausted"
15024 })
15025 );
15026 assert!(runner.state.merge.is_none());
15027 }
15028 }
15029
15030 #[test]
15034 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15035 crate::run::pin_test_home();
15036 let mut runner = runner_at(RunStatus::Implementing);
15037 let interrupt = Pause::new();
15038 runner.watch_interrupt(interrupt.clone());
15039
15040 interrupt.park_because("task a1b2 asked to run first");
15041
15042 assert!(runner.park_here().expect("park_here"));
15043 assert!(runner.state.parked);
15044 let last = runner.state.events.last().expect("a park event");
15045 assert_eq!(last.node, "park");
15046 assert!(
15047 last.message.contains("task a1b2 asked to run first"),
15048 "expected the interrupt reason in {:?}",
15049 last.message
15050 );
15051 }
15052
15053 #[test]
15056 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15057 crate::run::pin_test_home();
15058 let mut runner = runner_at(RunStatus::Implementing);
15059 runner.state.begin_pass();
15060 let shutdown = Pause::new();
15061 runner.on_pause(shutdown.clone());
15062 let upgrade = ParkCause::Upgrade {
15063 from: "0.121.0".to_owned(),
15064 to: Some("0.121.1".to_owned()),
15065 };
15066 shutdown.park_for(upgrade.clone());
15067 shutdown.park_for(ParkCause::Operator);
15068
15069 assert!(runner.park_here().expect("park_here"));
15070 assert_eq!(
15071 runner.state.passes[0].end,
15072 Some(crate::run::PassEnd::Parked { cause: upgrade })
15073 );
15074 let last = runner.state.events.last().expect("a park event");
15075 assert!(last.message.contains("0.121.0"), "{}", last.message);
15076 }
15077
15078 #[tokio::test]
15079 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15080 crate::run::pin_test_home();
15081 let mut runner = runner_at(RunStatus::Reviewing);
15082 runner.state.save().unwrap();
15083 let mut execution = runner.claim_driver().await.expect("first execution");
15084 assert_eq!(execution.runner.state.passes.len(), 1);
15085 execution
15086 .driver
15087 .finish(&mut execution.runner.state)
15088 .await
15089 .unwrap();
15090 }
15091
15092 #[tokio::test]
15093 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15094 crate::run::pin_test_home();
15095 let mut runner = runner_at(RunStatus::Reviewing);
15096 runner.state.config.agents = vec![conductor()];
15097 runner.state.config.graph.timeout_resource_wait = 0;
15098 runner.state.save().unwrap();
15099 let id = runner.state.id.clone();
15100 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15101 assert!(runner.claim_driver().await.is_err());
15102 drop(owner);
15103 drop(runner);
15104 let mut resumed = Runner::resume(&id).unwrap();
15105 assert_eq!(resumed.state.driver_claimed, Some(false));
15106 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15107 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15108 let mut execution = resumed
15109 .claim_driver()
15110 .await
15111 .expect("retry initial claim after reload");
15112 assert_eq!(execution.runner.state.driver_claimed, Some(true));
15113 assert_eq!(execution.runner.state.passes.len(), 1);
15114 execution
15115 .driver
15116 .finish(&mut execution.runner.state)
15117 .await
15118 .unwrap();
15119 }
15120
15121 #[tokio::test]
15122 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15123 crate::run::pin_test_home();
15124 let mut runner = runner_at(RunStatus::Reviewing);
15125 runner.state.config.agents = vec![conductor()];
15126 runner.state.save().unwrap();
15127 let id = runner.state.id.clone();
15128 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15129 let mut resumed = Runner::resume(&id).unwrap();
15130 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15131 assert!(resumed.claim_driver().await.is_err());
15132 assert!(RunState::load(&id).unwrap().passes.is_empty());
15133 drop(lease);
15134 let mut execution = resumed.claim_driver().await.unwrap();
15135 execution
15136 .driver
15137 .finish(&mut execution.runner.state)
15138 .await
15139 .unwrap();
15140 }
15141
15142 #[tokio::test]
15143 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15144 crate::run::pin_test_home();
15145 let mut runner = runner_at(RunStatus::Reviewing);
15146 let state = &mut runner.state;
15147 state.config.agents = vec![conductor()];
15148 state.driver_claimed = None;
15149 state.driver_pid = Some(std::process::id());
15150 state.driver_started_at = None;
15151 state.begin_pass();
15152 state.save().unwrap();
15153 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15154 std::fs::write(
15155 state.dir().join("driver.json"),
15156 serde_json::to_vec(&serde_json::json!({
15157 "pid": state.driver_pid, "started_at": null,
15158 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15159 }))
15160 .unwrap(),
15161 )
15162 .unwrap();
15163 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15164 drop(lease);
15165 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15166 let mut resumed = Runner::resume(&state.id).unwrap();
15167 assert!(!resumed.state.driver_exited);
15168 let mut execution = resumed
15169 .claim_driver()
15170 .await
15171 .expect("lease proves previous execution ended");
15172 assert_eq!(execution.runner.state.passes.len(), 2);
15173 execution
15174 .driver
15175 .finish(&mut execution.runner.state)
15176 .await
15177 .unwrap();
15178 }
15179
15180 #[tokio::test]
15181 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15182 crate::run::pin_test_home();
15183 let mut runner = runner_at(RunStatus::Reviewing);
15184 let mut json = serde_json::to_value(&runner.state).unwrap();
15185 json.as_object_mut().unwrap().remove("driver_claimed");
15186 runner.state = serde_json::from_value(json).unwrap();
15187 assert_eq!(runner.state.driver_claimed, None);
15188 runner.state.save().unwrap();
15189 let mut execution = runner
15190 .claim_driver()
15191 .await
15192 .expect("the execution lease, not a missing marker, excludes a second driver");
15193 assert_eq!(execution.runner.state.passes.len(), 1);
15194 execution
15195 .driver
15196 .finish(&mut execution.runner.state)
15197 .await
15198 .unwrap();
15199 }
15200
15201 #[tokio::test]
15202 async fn driver_admission_refuses_a_proven_live_driver() {
15203 crate::run::pin_test_home();
15204 let mut runner = runner_at(RunStatus::Reviewing);
15205 let pid = std::process::id();
15206 runner.state.driver_pid = Some(pid);
15207 runner.state.driver_started_at = crate::proc::process_started_at(pid);
15208 assert!(runner.state.driver_started_at.is_some());
15209 runner.state.save().unwrap();
15210 let error = runner
15211 .claim_driver()
15212 .await
15213 .err()
15214 .expect("a live driver must refuse");
15215 assert!(error.to_string().contains("live driver"), "{error:#}");
15216 let saved = RunState::load(&runner.state.id).unwrap();
15217 assert_eq!(saved.driver_pid, Some(pid));
15218 assert_eq!(saved.status, RunStatus::Reviewing);
15219 assert!(saved.passes.is_empty());
15220 }
15221
15222 #[test]
15223 fn parking_resource_halt_persists_the_pass_once() {
15224 crate::run::pin_test_home();
15225 let mut runner = runner_at(RunStatus::Reviewing);
15226 runner.state.begin_pass();
15227 runner.state.resource_block = Some(crate::run::ResourceBlock {
15228 node: "fix".into(),
15229 kind: "shared build cache".into(),
15230 detail: "cancelled".into(),
15231 at: Timestamp::now(),
15232 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15233 });
15234 let cause = ParkCause::Upgrade {
15235 from: "old".into(),
15236 to: Some("new".into()),
15237 };
15238 runner.pause.park_for(cause.clone());
15239 runner.stop_for_resource().unwrap();
15242 assert!(runner.resource_halt().unwrap());
15243 assert!(runner.park_here().unwrap());
15244 let saved = RunState::load(&runner.state.id).unwrap();
15245 assert!(saved.parked);
15246 assert_eq!(saved.status, RunStatus::Reviewing);
15247 assert!(saved.passes[0].ended_at.is_some());
15248 assert_eq!(
15249 saved.passes[0].end,
15250 Some(crate::run::PassEnd::Parked {
15251 cause: cause.clone()
15252 })
15253 );
15254 assert_eq!(
15255 saved.resource_block.unwrap().cause,
15256 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15257 );
15258 assert_eq!(
15259 saved
15260 .events
15261 .iter()
15262 .filter(|event| event.node == "park")
15263 .count(),
15264 1
15265 );
15266 }
15267
15268 #[test]
15276 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15277 crate::run::pin_test_home();
15278 let mut runner = runner_at(RunStatus::Implementing);
15279 let shutdown = Pause::new();
15280 runner.on_pause(shutdown.clone());
15281 let interrupt = Pause::new();
15282 runner.watch_interrupt(interrupt.clone());
15283
15284 assert!(!runner.park_here().expect("park_here"));
15286 assert!(!runner.state.parked);
15287
15288 interrupt.park_because("test");
15290 assert!(!shutdown.parked());
15291 assert!(runner.park_here().expect("park_here"));
15292 }
15293
15294 #[tokio::test]
15308 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15309 crate::run::pin_test_home();
15310 let mut runner = runner_at(RunStatus::Implementing);
15311 let interrupt = Pause::new();
15312 runner.watch_interrupt(interrupt.clone());
15313
15314 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15315 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15316
15317 let node = async move {
15321 started_tx.send(()).expect("send started");
15322 finish_rx.await.expect("recv finish");
15323 "node finished"
15324 };
15325
15326 let interrupter = async move {
15327 started_rx.await.expect("recv started");
15328 interrupt.park_because("higher-priority task waiting");
15330 tokio::task::yield_now().await;
15334 finish_tx.send(()).expect("send finish");
15335 };
15336
15337 let (node_result, ()) = tokio::join!(node, interrupter);
15338 assert_eq!(
15339 node_result, "node finished",
15340 "the in-flight call ran to completion"
15341 );
15342
15343 assert!(runner.park_here().expect("park_here"));
15346 assert!(runner.state.parked);
15347 }
15348
15349 #[test]
15355 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15356 crate::run::pin_test_home();
15357 let mut runner = runner_at(RunStatus::Judging);
15358 runner.state.config.agents = vec![conductor()];
15362 runner.state.candidates = vec![Candidate {
15363 index: 0,
15364 label: 'A',
15365 agent: "alpha".to_owned(),
15366 branch: "magi/x/A".to_owned(),
15367 worktree: PathBuf::from("/nonexistent/worktree"),
15368 summary: "did the thing".to_owned(),
15369 stat: "1 file changed".to_owned(),
15370 files: 1,
15371 commits: 1,
15372 empty: false,
15373 failed: None,
15374 verified_noop: None,
15375 duration_ms: 1234,
15376 folded: false,
15377 }];
15378 let run_id = runner.state.id.clone();
15379
15380 let interrupt = Pause::new();
15381 runner.watch_interrupt(interrupt.clone());
15382 interrupt.park_because("task c3d4 asked to run first");
15383 assert!(runner.park_here().expect("park_here"));
15384
15385 let resumed = Runner::resume(&run_id).expect("resume");
15386 assert_eq!(resumed.state.candidates.len(), 1);
15387 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15388 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15389 assert_eq!(resumed.state.status, runner.state.status);
15390 assert!(
15391 resumed.state.parked,
15392 "still parked until `execute` actually walks the graph again"
15393 );
15394 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15395 }
15396
15397 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15399 let mut q = ask::Question::new(
15400 run.to_owned(),
15401 "implement".to_owned(),
15402 "impl-A".to_owned(),
15403 "Which storage backend should the cache use?".to_owned(),
15404 String::new(),
15405 vec!["SQLite".to_owned(), "Redis".to_owned()],
15406 );
15407 store.put(&mut q).unwrap();
15408 q
15409 }
15410
15411 #[test]
15412 fn a_failed_runs_open_question_is_abandoned() {
15413 ask_test_home();
15414 let store = ask::Questions::open();
15415 let mut runner = runner_at(RunStatus::Failed);
15416 let run = runner.state.id.clone();
15417 let q = ask_open_question(&store, &run);
15418
15419 runner.settle_questions();
15420
15421 let back = store.get(&q.id).unwrap();
15422 assert!(
15423 !back.status.open(),
15424 "the seat that asked died with the run; nobody is left to read an answer"
15425 );
15426 assert!(
15427 back.detail.contains(&run) && back.detail.contains("failed"),
15428 "the reason names what the run became, not just that it is gone: {}",
15429 back.detail
15430 );
15431 }
15432
15433 #[test]
15434 fn a_merged_runs_open_question_is_abandoned_too() {
15435 ask_test_home();
15436 let store = ask::Questions::open();
15437 for status in [RunStatus::Merged, RunStatus::Ready] {
15440 let mut runner = runner_at(status);
15441 let run = runner.state.id.clone();
15442 let q = ask_open_question(&store, &run);
15443
15444 runner.settle_questions();
15445
15446 let back = store.get(&q.id).unwrap();
15447 assert!(
15448 !back.status.open(),
15449 "{status:?} run's question must not outlive the run"
15450 );
15451 }
15452 }
15453
15454 #[test]
15455 fn a_still_resumable_runs_open_question_is_left_alone() {
15456 ask_test_home();
15457 let store = ask::Questions::open();
15458 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15464 let mut runner = runner_at(status);
15465 let run = runner.state.id.clone();
15466 let q = ask_open_question(&store, &run);
15467
15468 runner.settle_questions();
15469
15470 let back = store.get(&q.id).unwrap();
15471 assert!(
15472 back.status.open(),
15473 "{status:?} is still alive; the question must still be waiting"
15474 );
15475 }
15476 }
15477
15478 #[test]
15479 fn settle_questions_never_touches_an_already_answered_question() {
15480 ask_test_home();
15481 let store = ask::Questions::open();
15482 let mut runner = runner_at(RunStatus::Failed);
15483 let run = runner.state.id.clone();
15484 let mut q = ask_open_question(&store, &run);
15485 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15486 .unwrap();
15487 store.put(&mut q).unwrap();
15488
15489 runner.settle_questions();
15494 runner.settle_questions();
15495
15496 let back = store.get(&q.id).unwrap();
15497 assert_eq!(
15498 back.status,
15499 ask::QuestionStatus::Answered,
15500 "a real answer is a decision on record, never overwritten by a sweep"
15501 );
15502 }
15503
15504 #[tokio::test]
15515 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15516 crate::run::pin_test_home();
15517 let tmp = tempfile::tempdir().expect("tempdir");
15518 let repo = tmp.path().join("repo");
15519 std::fs::create_dir_all(&repo).unwrap();
15520 init_repo(&repo);
15521
15522 let mut config = Config::default();
15523 config.graph.worktree_root = Some(tmp.path().join("wt"));
15524
15525 let mut state = RunState::new(
15526 repo.clone(),
15527 "main".to_owned(),
15528 "deadbeef".to_owned(),
15529 "task".to_owned(),
15530 config,
15531 );
15532 let root = state.worktree_root();
15533 let wt_a = root.join("cand-A");
15534 let wt_b = root.join("cand-B");
15535 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15536 .await
15537 .expect("worktree A");
15538 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15539 .await
15540 .expect("worktree B");
15541
15542 state.candidates = vec![
15543 Candidate {
15544 index: 0,
15545 label: 'A',
15546 agent: "alpha".to_owned(),
15547 branch: "magi/x/A".to_owned(),
15548 worktree: wt_a.clone(),
15549 summary: String::new(),
15550 stat: String::new(),
15551 files: 0,
15552 commits: 0,
15553 empty: false,
15554 failed: None,
15555 verified_noop: None,
15556 duration_ms: 0,
15557 folded: false,
15558 },
15559 Candidate {
15560 index: 1,
15561 label: 'B',
15562 agent: "beta".to_owned(),
15563 branch: "magi/x/B".to_owned(),
15564 worktree: wt_b.clone(),
15565 summary: String::new(),
15566 stat: String::new(),
15567 files: 0,
15568 commits: 0,
15569 empty: false,
15570 failed: None,
15571 verified_noop: None,
15572 duration_ms: 0,
15573 folded: false,
15574 },
15575 ];
15576 state.tally = Some(Tally {
15577 first_choice: BTreeMap::from([('A', 1)]),
15578 borda: BTreeMap::new(),
15579 winner: 'A',
15580 rankings: 1,
15581 unanimous_initial: true,
15582 deliberated: false,
15583 changed_votes: 0,
15584 unanimous_final: true,
15585 tie_break: None,
15586 judges: 1,
15587 present: 1,
15588 quorum: 1,
15589 met_quorum: true,
15590 uncontested: None,
15591 });
15592 state.status = RunStatus::Ready;
15593
15594 fold_run(&mut state, false, &crate::run::home())
15595 .await
15596 .expect("fold_run");
15597
15598 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15599 assert!(
15600 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15601 "the unmerged winner's branch survives"
15602 );
15603 assert!(
15604 !state.candidates[0].folded,
15605 "the winner is not marked folded"
15606 );
15607
15608 assert!(!wt_b.exists(), "the loser's worktree is removed");
15609 assert!(
15610 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15611 "the loser's branch is removed"
15612 );
15613 assert!(state.candidates[1].folded, "the loser is marked folded");
15614 }
15615
15616 #[tokio::test]
15617 #[cfg(unix)]
15618 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15619 use std::os::fd::AsRawFd as _;
15620 use std::time::Duration;
15621
15622 let tmp = tempfile::tempdir().unwrap();
15623 let repo = tmp.path().join("repo");
15624 std::fs::create_dir_all(&repo).unwrap();
15625 init_repo(&repo);
15626 let common = repo.join(".git");
15627 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15628 let home = tmp.path().join("home");
15629 let mut config = Config::default();
15630 config.coordination = crate::config::test_coordination();
15631 config.graph.worktree_root = Some(tmp.path().join("wt"));
15632 config.graph.timeout_resource_wait = 1;
15633 let mut state = RunState::new(
15634 repo.clone(),
15635 "main".into(),
15636 "deadbeef".into(),
15637 "task".into(),
15638 config,
15639 );
15640 let root = state.worktree_root();
15641 let candidate = root.join("cand-A");
15642 let reviewer = root.join("review-1");
15643 let branch = state.branch_for('A');
15644 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15645 .await
15646 .unwrap();
15647 git::worktree_add_detached(&repo, &reviewer, "main")
15648 .await
15649 .unwrap();
15650 git::acquire_worktree_config(&repo).await.unwrap();
15651 state.enabled_worktree_config = true;
15652 state.status = RunStatus::Ready;
15653 state.candidates.push(Candidate {
15654 index: 0,
15655 label: 'A',
15656 agent: "implement".into(),
15657 branch: branch.clone(),
15658 worktree: candidate.clone(),
15659 summary: String::new(),
15660 stat: String::new(),
15661 files: 0,
15662 commits: 0,
15663 empty: false,
15664 failed: None,
15665 verified_noop: None,
15666 duration_ms: 0,
15667 folded: false,
15668 });
15669 state.save_under(&home).unwrap();
15670 let config_before = std::fs::read(common.join("config")).unwrap();
15671 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15672 .await
15673 .unwrap();
15674 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15677 assert_eq!(
15678 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15679 0
15680 );
15681 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15682 assert!(
15683 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15684 .await
15685 .is_err()
15686 );
15687 assert!(candidate.exists());
15688 assert!(reviewer.exists());
15689 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15690 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15691 assert_eq!(
15692 git::git(&repo, &["worktree", "list", "--porcelain"])
15693 .await
15694 .unwrap(),
15695 worktrees_before
15696 );
15697 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15698 .await
15699 .unwrap()
15700 .unwrap_err();
15701 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15702 drop(fold);
15703 assert!(candidate.exists() && reviewer.exists());
15704 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15705 assert_eq!(
15706 state.resource_block.as_ref().unwrap().kind,
15707 "repository coordination"
15708 );
15709 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15710 drop(external);
15711 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15712 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15713 .await
15714 .unwrap()
15715 .unwrap();
15716 drop(fold);
15717 assert!(removed.contains(&branch));
15718 assert!(!root.exists());
15719 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15720 assert!(state.candidates[0].folded);
15721 assert!(!state.enabled_worktree_config);
15722 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15723 let enabled = git::git_raw(
15724 &repo,
15725 &["config", "--local", "--get", "extensions.worktreeConfig"],
15726 )
15727 .await
15728 .unwrap();
15729 assert!(!enabled.ok(), "the last config reference was released");
15730 }
15731
15732 #[tokio::test]
15735 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15736 let tmp = tempfile::tempdir().expect("tempdir");
15737 let repo = tmp.path().join("repo");
15738 std::fs::create_dir_all(&repo).unwrap();
15739 init_repo(&repo);
15740 let home = tmp.path().join("home");
15741
15742 let mut config = Config::default();
15743 config.graph.worktree_root = Some(tmp.path().join("wt"));
15744 let mut state = RunState::new(
15745 repo.clone(),
15746 "main".to_owned(),
15747 "deadbeef".to_owned(),
15748 "task".to_owned(),
15749 config,
15750 );
15751 git::git(&repo, &["branch", "magi/x/A", "main"])
15753 .await
15754 .expect("branch");
15755 state.candidates = vec![Candidate {
15756 index: 0,
15757 label: 'A',
15758 agent: "alpha".to_owned(),
15759 branch: "magi/x/A".to_owned(),
15760 worktree: state.worktree_root().join("cand-A"),
15761 summary: String::new(),
15762 stat: String::new(),
15763 files: 0,
15764 commits: 0,
15765 empty: false,
15766 failed: None,
15767 verified_noop: None,
15768 duration_ms: 0,
15769 folded: true,
15770 }];
15771 state.released_to = Some("20260901-000000-new1".to_owned());
15772 state.released_branches = vec!["magi/x/A".to_owned()];
15773
15774 fold_run(&mut state, true, &home).await.expect("fold_run");
15775 state.status = RunStatus::Merged;
15776 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15777 state.merge = Some(crate::run::MergeOutcome {
15778 mode: MergeMode::Pr,
15779 ok: true,
15780 detail: "confirmed merged".into(),
15781 empty: false,
15782 });
15783 fold_run(&mut state, true, &home)
15784 .await
15785 .expect("merged fold");
15786
15787 assert!(
15788 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15789 "the handed-over branch survives a fold"
15790 );
15791 }
15792
15793 #[tokio::test]
15797 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15798 let tmp = tempfile::tempdir().expect("tempdir");
15799 let repo = tmp.path().join("repo");
15800 std::fs::create_dir_all(&repo).unwrap();
15801 init_repo(&repo);
15802 let run = |args: &[&str]| {
15803 let out = std::process::Command::new("git")
15804 .quiet()
15805 .args(args)
15806 .current_dir(&repo)
15807 .output()
15808 .expect("spawn git");
15809 assert!(out.status.success(), "git {args:?}");
15810 };
15811 run(&["branch", "magi/x/A"]);
15812 run(&["checkout", "-q", "-b", "magi/x/B"]);
15813 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15814 run(&["add", "-A"]);
15815 run(&["commit", "-q", "-m", "work"]);
15816 run(&["checkout", "-q", "main"]);
15817
15818 let probe = RunState::new(
15822 repo.clone(),
15823 "main".to_owned(),
15824 "deadbeef".to_owned(),
15825 "task".to_owned(),
15826 Config::default(),
15827 );
15828 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15829 let bare = tmp.path().join("origin.git");
15830 let out = std::process::Command::new("git")
15831 .quiet()
15832 .args(["init", "-q", "--bare"])
15833 .arg(&bare)
15834 .output()
15835 .expect("spawn git");
15836 assert!(out.status.success(), "git init --bare");
15837 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15838 run(&["push", "-q", "origin", "main"]);
15839
15840 let mut state = RunState::new(
15841 repo.clone(),
15842 "main".to_owned(),
15843 "deadbeef".to_owned(),
15844 "task".to_owned(),
15845 Config::default(),
15846 );
15847 state.seeds = vec![refs::Seed {
15848 token: "magi/27b2/A".to_owned(),
15849 kind: refs::SeedKind::Unresolved,
15850 sha: String::new(),
15851 branch: true,
15852 detail: "no branch or commit named magi/27b2/A".to_owned(),
15853 }];
15854
15855 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15856 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15857 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15858 let detail = empty_candidate_detail(&state, "main");
15859 assert!(detail.starts_with("empty candidate"), "{detail}");
15860 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15861 assert!(detail.contains("--start-from"), "{detail}");
15862 }
15863
15864 #[tokio::test]
15873 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15874 let tmp = tempfile::tempdir().expect("tempdir");
15875 let repo = tmp.path().join("repo");
15876 std::fs::create_dir_all(&repo).unwrap();
15877 init_repo(&repo);
15878
15879 let mut config = Config::default();
15880 config.merge.mode = MergeMode::Local;
15881
15882 let mut state = RunState::new(
15883 repo.clone(),
15884 "main".to_owned(),
15885 "deadbeef".to_owned(),
15886 "task".to_owned(),
15887 config,
15888 );
15889 state.candidates = vec![Candidate {
15890 index: 0,
15891 label: 'A',
15892 agent: "alpha".to_owned(),
15893 branch: "does-not-exist".to_owned(),
15894 worktree: repo.clone(),
15895 summary: String::new(),
15896 stat: String::new(),
15897 files: 0,
15898 commits: 0,
15899 empty: false,
15900 failed: None,
15901 verified_noop: None,
15902 duration_ms: 0,
15903 folded: false,
15904 }];
15905 state.tally = Some(Tally {
15906 first_choice: BTreeMap::from([('A', 1)]),
15907 borda: BTreeMap::new(),
15908 winner: 'A',
15909 rankings: 1,
15910 unanimous_initial: true,
15911 deliberated: false,
15912 changed_votes: 0,
15913 unanimous_final: true,
15914 tie_break: None,
15915 judges: 0,
15916 present: 0,
15917 quorum: 0,
15918 met_quorum: true,
15919 uncontested: Some("only candidate A produced a change".to_owned()),
15920 });
15921 state.reviews = vec![ReviewRound {
15922 round: 1,
15923 head: "deadbeef".to_owned(),
15924 verified_head: None,
15925 verified_at: None,
15926 reviews: Vec::new(),
15927 e2e: Vec::new(),
15928 fix: None,
15929 blocking: 0,
15930 answered: 0,
15931 expected: 0,
15932 clean: true,
15933 verify_retried: false,
15934 e2e_deferred: false,
15935 e2e_defer_reason: None,
15936 progressed: false,
15937 vote_split: false,
15938 reconsideration: Vec::new(),
15939 verdict: None,
15940 }];
15941 state.gate = vec![CommandOutcome {
15942 command: "test".to_owned(),
15943 code: Some(0),
15944 output_tail: String::new(),
15945 duration_ms: 0,
15946 resource_blocked: false,
15947 }];
15948 state.gate_ran = true;
15949 state.status = RunStatus::Ready;
15954 state.merge = Some(MergeOutcome {
15955 mode: MergeMode::Local,
15956 ok: false,
15957 detail: "already concluded".to_owned(),
15958 empty: false,
15959 });
15960
15961 let mut runner = Runner {
15962 state,
15963 roles: ResolvedRoles {
15964 implementers: Vec::new(),
15965 judges: Vec::new(),
15966 reviewers: Vec::new(),
15967 fixer: None,
15968 conductor: conductor(),
15969 implementer_roster: Vec::new(),
15970 judge_roster: Vec::new(),
15971 reviewer_roster: Vec::new(),
15972 },
15973 sem: Arc::new(Semaphore::new(1)),
15974 pause: Pause::new(),
15975 interrupt: Pause::new(),
15976 };
15977
15978 runner.merge().await.expect("merge");
15979
15980 assert_eq!(
15981 runner.state.status,
15982 RunStatus::Ready,
15983 "a concluded run's status must not change on reentry"
15984 );
15985 assert_eq!(
15986 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15987 Some("already concluded"),
15988 "merge must not run again once the node already recorded an outcome"
15989 );
15990 }
15991
15992 #[tokio::test]
16001 async fn merge_refuses_a_gate_that_has_not_actually_run() {
16002 let tmp = tempfile::tempdir().expect("tempdir");
16003 let repo = tmp.path().join("repo");
16004 std::fs::create_dir_all(&repo).unwrap();
16005 init_repo(&repo);
16006
16007 let mut config = Config::default();
16008 config.merge.mode = MergeMode::Local;
16009
16010 let mut state = RunState::new(
16011 repo.clone(),
16012 "main".to_owned(),
16013 "deadbeef".to_owned(),
16014 "task".to_owned(),
16015 config,
16016 );
16017 state.candidates = vec![Candidate {
16018 index: 0,
16019 label: 'A',
16020 agent: "alpha".to_owned(),
16021 branch: "does-not-exist".to_owned(),
16022 worktree: repo.clone(),
16023 summary: String::new(),
16024 stat: String::new(),
16025 files: 0,
16026 commits: 0,
16027 empty: false,
16028 failed: None,
16029 verified_noop: None,
16030 duration_ms: 0,
16031 folded: false,
16032 }];
16033 state.tally = Some(Tally {
16034 first_choice: BTreeMap::from([('A', 1)]),
16035 borda: BTreeMap::new(),
16036 winner: 'A',
16037 rankings: 1,
16038 unanimous_initial: true,
16039 deliberated: false,
16040 changed_votes: 0,
16041 unanimous_final: true,
16042 tie_break: None,
16043 judges: 0,
16044 present: 0,
16045 quorum: 0,
16046 met_quorum: true,
16047 uncontested: Some("only candidate A produced a change".to_owned()),
16048 });
16049 state.reviews = vec![ReviewRound {
16050 round: 1,
16051 head: "deadbeef".to_owned(),
16052 verified_head: None,
16053 verified_at: None,
16054 reviews: Vec::new(),
16055 e2e: Vec::new(),
16056 fix: None,
16057 blocking: 0,
16058 answered: 0,
16059 expected: 0,
16060 clean: true,
16061 verify_retried: false,
16062 e2e_deferred: false,
16063 e2e_defer_reason: None,
16064 progressed: false,
16065 vote_split: false,
16066 reconsideration: Vec::new(),
16067 verdict: None,
16068 }];
16069 state.gate = Vec::new();
16071 state.gate_ran = false;
16072 state.status = RunStatus::Gating;
16073
16074 let mut runner = Runner {
16075 state,
16076 roles: ResolvedRoles {
16077 implementers: Vec::new(),
16078 judges: Vec::new(),
16079 reviewers: Vec::new(),
16080 fixer: None,
16081 conductor: conductor(),
16082 implementer_roster: Vec::new(),
16083 judge_roster: Vec::new(),
16084 reviewer_roster: Vec::new(),
16085 },
16086 sem: Arc::new(Semaphore::new(1)),
16087 pause: Pause::new(),
16088 interrupt: Pause::new(),
16089 };
16090
16091 runner.merge().await.expect("merge");
16092
16093 assert!(
16094 runner.state.merge.is_none(),
16095 "an empty gate must never be read as a passing one: {:?}",
16096 runner.state.merge
16097 );
16098 }
16099
16100 #[tokio::test]
16107 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16108 let tmp = tempfile::tempdir().expect("tempdir");
16109 let repo = tmp.path().join("repo");
16110 std::fs::create_dir_all(&repo).unwrap();
16111 init_repo(&repo);
16112
16113 let config = Config::default();
16115
16116 let mut state = RunState::new(
16117 repo.clone(),
16118 "main".to_owned(),
16119 "deadbeef".to_owned(),
16120 "task".to_owned(),
16121 config,
16122 );
16123 state.candidates = vec![Candidate {
16124 index: 0,
16125 label: 'A',
16126 agent: "alpha".to_owned(),
16127 branch: "does-not-exist".to_owned(),
16128 worktree: repo.clone(),
16129 summary: String::new(),
16130 stat: String::new(),
16131 files: 0,
16132 commits: 0,
16133 empty: false,
16134 failed: None,
16135 verified_noop: None,
16136 duration_ms: 0,
16137 folded: false,
16138 }];
16139 state.tally = Some(Tally {
16140 first_choice: BTreeMap::from([('A', 1)]),
16141 borda: BTreeMap::new(),
16142 winner: 'A',
16143 rankings: 1,
16144 unanimous_initial: true,
16145 deliberated: false,
16146 changed_votes: 0,
16147 unanimous_final: true,
16148 tie_break: None,
16149 judges: 0,
16150 present: 0,
16151 quorum: 0,
16152 met_quorum: true,
16153 uncontested: Some("only candidate A produced a change".to_owned()),
16154 });
16155 state.reviews = vec![ReviewRound {
16156 round: 1,
16157 head: "deadbeef".to_owned(),
16158 verified_head: None,
16159 verified_at: None,
16160 reviews: Vec::new(),
16161 e2e: Vec::new(),
16162 fix: None,
16163 blocking: 0,
16164 answered: 0,
16165 expected: 0,
16166 clean: true,
16167 verify_retried: false,
16168 e2e_deferred: false,
16169 e2e_defer_reason: None,
16170 progressed: false,
16171 vote_split: false,
16172 reconsideration: Vec::new(),
16173 verdict: None,
16174 }];
16175
16176 let mut runner = Runner {
16177 state,
16178 roles: ResolvedRoles {
16179 implementers: Vec::new(),
16180 judges: Vec::new(),
16181 reviewers: Vec::new(),
16182 fixer: None,
16183 conductor: conductor(),
16184 implementer_roster: Vec::new(),
16185 judge_roster: Vec::new(),
16186 reviewer_roster: Vec::new(),
16187 },
16188 sem: Arc::new(Semaphore::new(1)),
16189 pause: Pause::new(),
16190 interrupt: Pause::new(),
16191 };
16192
16193 runner.gate().await.expect("gate");
16194 assert!(
16195 runner.state.gate_ran,
16196 "zero configured commands is still a real attempt, not an unrun gate"
16197 );
16198 assert!(runner.state.gate.is_empty());
16199 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16200 assert_ne!(
16201 runner.state.status,
16202 RunStatus::Blocked,
16203 "a gate with nothing to check must not read as failed"
16204 );
16205
16206 runner.merge().await.expect("merge");
16207 assert_eq!(
16208 runner.state.status,
16209 RunStatus::Ready,
16210 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16211 );
16212 }
16213
16214 #[tokio::test]
16225 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16226 crate::run::pin_test_home();
16227 let home = crate::run::home();
16228
16229 let tmp = tempfile::tempdir().expect("tempdir");
16230 let repo = tmp.path().join("repo");
16231 std::fs::create_dir_all(&repo).unwrap();
16232 init_repo(&repo);
16233 let cache_dir = tmp.path().join("target");
16236
16237 let mut config = Config::default();
16238 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16239 config.graph.timeout_verify = Some(2);
16242 config.graph.timeout_resource_wait = 2;
16243
16244 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16245 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16246 .expect("no io error acquiring directly")
16247 {
16248 crate::cache::AcquireOutcome::Acquired(g) => g,
16249 crate::cache::AcquireOutcome::Busy(b) => {
16250 panic!("expected the direct acquire to win the lease first: {b:?}")
16251 }
16252 };
16253
16254 let mut state = RunState::new(
16255 repo.clone(),
16256 "main".to_owned(),
16257 "deadbeef".to_owned(),
16258 "task".to_owned(),
16259 config,
16260 );
16261 state.candidates = vec![Candidate {
16262 index: 0,
16263 label: 'A',
16264 agent: "alpha".to_owned(),
16265 branch: "does-not-exist".to_owned(),
16266 worktree: repo.clone(),
16267 summary: String::new(),
16268 stat: String::new(),
16269 files: 0,
16270 commits: 0,
16271 empty: false,
16272 failed: None,
16273 verified_noop: None,
16274 duration_ms: 0,
16275 folded: false,
16276 }];
16277 state.tally = Some(Tally {
16278 first_choice: BTreeMap::from([('A', 1)]),
16279 borda: BTreeMap::new(),
16280 winner: 'A',
16281 rankings: 1,
16282 unanimous_initial: true,
16283 deliberated: false,
16284 changed_votes: 0,
16285 unanimous_final: true,
16286 tie_break: None,
16287 judges: 0,
16288 present: 0,
16289 quorum: 0,
16290 met_quorum: true,
16291 uncontested: Some("only candidate A produced a change".to_owned()),
16292 });
16293 state.reviews = vec![ReviewRound {
16294 round: 1,
16295 head: "deadbeef".to_owned(),
16296 verified_head: None,
16297 verified_at: None,
16298 reviews: Vec::new(),
16299 e2e: Vec::new(),
16300 fix: None,
16301 blocking: 0,
16302 answered: 0,
16303 expected: 0,
16304 clean: true,
16305 verify_retried: false,
16306 e2e_deferred: false,
16307 e2e_defer_reason: None,
16308 progressed: false,
16309 vote_split: false,
16310 reconsideration: Vec::new(),
16311 verdict: None,
16312 }];
16313
16314 let mut runner = Runner {
16315 state,
16316 roles: ResolvedRoles {
16317 implementers: Vec::new(),
16318 judges: Vec::new(),
16319 reviewers: Vec::new(),
16320 fixer: None,
16321 conductor: conductor(),
16322 implementer_roster: Vec::new(),
16323 judge_roster: Vec::new(),
16324 reviewer_roster: Vec::new(),
16325 },
16326 sem: Arc::new(Semaphore::new(1)),
16327 pause: Pause::new(),
16328 interrupt: Pause::new(),
16329 };
16330
16331 let started = std::time::Instant::now();
16332 runner.gate().await.expect("gate");
16333 assert!(
16334 started.elapsed() < Duration::from_secs(1),
16335 "a gate with nothing to run must never wait on a lease it never needed"
16336 );
16337 assert!(
16338 runner.state.gate_ran,
16339 "zero commands is still a real, immediate attempt"
16340 );
16341 assert!(runner.state.gate.is_empty());
16342 assert_ne!(
16343 runner.state.status,
16344 RunStatus::Blocked,
16345 "must not read as resource-blocked on a lease it never asked for"
16346 );
16347 }
16348
16349 #[cfg(unix)]
16352 #[tokio::test]
16353 async fn verification_is_standalone_inside_a_live_worker() {
16354 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
16355 if std::env::var_os(CHILD).is_none() {
16356 let home = tempfile::tempdir().unwrap();
16357 let generation = crate::worker::pin_current(home.path()).unwrap();
16358 let output = std::process::Command::new(std::env::current_exe().unwrap())
16359 .args([
16360 "--exact",
16361 "graph::tests::verification_is_standalone_inside_a_live_worker",
16362 "--nocapture",
16363 ])
16364 .env(CHILD, "1")
16365 .env("MAGI_HOME", home.path())
16366 .env("MAGI_WORKER_HOME", home.path())
16367 .env("MAGI_WORKER_ID", "abcdef")
16368 .env("MAGI_WORKER_GENERATION", &generation.id)
16369 .env("MAGI_WORKER_EXE", &generation.executable)
16370 .env("MAGI_RUN", "verification-parent")
16371 .env("MAGI_NODE", "gate")
16372 .env("MAGI_SEAT", "worker-fixture")
16373 .env("MAGI_PROCESS_OWNER", "fixture-owner")
16374 .env("MAGI_NO_AUTOUPDATE", "0")
16375 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
16376 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
16377 .env(
16378 "VERIFY_EXPECT_TOOLCHAIN",
16379 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
16380 )
16381 .env(
16382 "VERIFY_EXPECT_CACHE",
16383 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
16384 )
16385 .output()
16386 .unwrap();
16387 assert!(
16388 output.status.success()
16389 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16390 "{}\n{}",
16391 String::from_utf8_lossy(&output.stdout),
16392 String::from_utf8_lossy(&output.stderr)
16393 );
16394 return;
16395 }
16396 let home = crate::run::home();
16397 let directory = home.join("workers/jobs/abcdef");
16398 std::fs::create_dir_all(&directory).unwrap();
16399 let job = crate::worker::Job {
16400 id: "abcdef".into(),
16401 sequence: 1,
16402 kind: "task".into(),
16403 key: "task:verification".into(),
16404 payload: serde_json::json!({}),
16405 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
16406 version: Some(env!("CARGO_PKG_VERSION").into()),
16407 status: crate::worker::Status::Running,
16408 result: None,
16409 pid: Some(std::process::id()),
16410 created_at: 0,
16411 finished_at: None,
16412 };
16413 std::fs::write(
16414 directory.join("job.json"),
16415 serde_json::to_vec(&job).unwrap(),
16416 )
16417 .unwrap();
16418 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
16419 let assert_worker = || {
16420 assert_eq!(
16421 crate::worker::current_job(&home).unwrap().unwrap().id,
16422 job.id
16423 );
16424 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
16425 assert_eq!(
16426 crate::worker::helper_executable(&home).unwrap(),
16427 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
16428 );
16429 assert!(
16430 crate::updater::Checker::new(&crate::config::Update {
16431 mode: crate::config::UpdateMode::Notify,
16432 interval: None,
16433 auto_restart: false,
16434 })
16435 .is_none(),
16436 "the driver remains pinned, even while tests are standalone"
16437 );
16438 };
16439 assert_worker();
16440 let repo = home.join("repo");
16441 std::fs::create_dir_all(&repo).unwrap();
16442 init_repo(&repo);
16443 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
16446 test -z "${MAGI_WORKER_GENERATION+x}" &&
16447 test -z "${MAGI_WORKER_HOME+x}" &&
16448 test -z "${MAGI_WORKER_EXE+x}" &&
16449 test "$MAGI_HOME/repo" -ef . &&
16450 test "$MAGI_RUN" = verification-parent &&
16451 test "$MAGI_NODE" = gate &&
16452 test "$MAGI_SEAT" = worker-fixture &&
16453 test -n "$MAGI_PROCESS_OWNER" &&
16454 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
16455 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
16456 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
16457 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
16458 assert!(
16460 !std::process::Command::new("sh")
16461 .args(["-c", probe])
16462 .current_dir(&repo)
16463 .status()
16464 .unwrap()
16465 .success()
16466 );
16467 let unrelated_home = tempfile::tempdir().unwrap();
16468 assert!(
16469 crate::worker::current_job(unrelated_home.path()).is_err(),
16470 "an inherited worker id cannot be resolved in a unit fixture's home"
16471 );
16472 let mut runner = runner_at(RunStatus::Gating);
16473 runner.state.repo = repo.clone();
16474 runner.state.config.disk.min_free_bytes = 0;
16475 runner.state.config.coordination = crate::config::test_coordination();
16476 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
16477 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
16478 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
16479 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
16480 let winner = Candidate {
16481 index: 0,
16482 label: 'A',
16483 agent: "fixture".into(),
16484 branch: "main".into(),
16485 worktree: repo.clone(),
16486 summary: String::new(),
16487 stat: String::new(),
16488 files: 0,
16489 commits: 0,
16490 empty: false,
16491 failed: None,
16492 verified_noop: None,
16493 duration_ms: 0,
16494 folded: false,
16495 };
16496 let (e2e, retried, pids) = run_e2e_with_retry(
16497 &mut runner.state,
16498 &["sh".into(), "-c".into()],
16499 &[format!("{probe} && printf initial")],
16500 &repo,
16501 Duration::from_secs(10),
16502 "initial",
16503 )
16504 .await;
16505 assert!(!retried && pids.is_empty());
16506 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
16507 assert_eq!(e2e[0].output_tail, "initial");
16508 let gate = runner.run_gate(&winner).await.unwrap();
16509 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
16510 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
16513 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
16514 passed: false,
16515 outcomes: Vec::new(),
16516 });
16517 runner.state.save().unwrap();
16518 let mut resumed = RunState::load(&runner.state.id).unwrap();
16519 let wait = crate::repo_lock::WaitPolicy::default();
16520 assert!(matches!(
16521 check_gate_repair_e2e(&mut resumed, &winner, &wait)
16522 .await
16523 .unwrap(),
16524 GateFix::Retry
16525 ));
16526 let saved = RunState::load(&resumed.id).unwrap();
16527 let repair = saved.gate_repair_e2e.unwrap();
16528 assert!(repair.passed);
16529 assert_eq!(repair.outcomes.len(), 1);
16530 assert!(repair.outcomes[0].ok());
16531 let landing = run_landing_gate(&mut resumed, &winner, &wait)
16532 .await
16533 .unwrap();
16534 assert_eq!(landing.len(), 1);
16535 assert!(landing[0].ok(), "{landing:?}");
16536 assert_eq!(
16537 std::fs::read_to_string(repo.join("phases")).unwrap(),
16538 "pre\ngate\ne2e\npre\ngate\n"
16539 );
16540 assert_worker();
16541 }
16542
16543 #[cfg(unix)]
16544 #[tokio::test]
16545 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16546 crate::run::pin_test_home();
16547 let tmp = tempfile::tempdir().unwrap();
16548 let mut config = Config::default();
16549 config.disk.min_free_bytes = 0;
16550 let mut state = RunState::new(
16551 tmp.path().to_owned(),
16552 "main".into(),
16553 "base".into(),
16554 "verification evidence".into(),
16555 config,
16556 );
16557 let (outcomes, pids) = run_commands(
16558 &mut state,
16559 "gate",
16560 "gate",
16561 1,
16562 &["sh".into(), "-c".into()],
16563 &[
16564 "printf before-timeout; exec sleep 30".into(),
16565 "kill -TERM $$".into(),
16566 "exit 7".into(),
16567 "printf success".into(),
16568 ],
16569 tmp.path(),
16570 Duration::from_secs(1),
16571 )
16572 .await;
16573 wait_for_timed_out_children_to_die(&pids).await;
16574 assert_eq!(
16575 pids.len(),
16576 1,
16577 "unleased timeout retains its death-check PID"
16578 );
16579 assert_eq!(outcomes.len(), 4);
16580 assert_eq!(outcomes[0].code, None);
16581 assert!(outcomes[0].output_tail.contains("before-timeout"));
16582 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16583 assert_eq!(outcomes[1].code, None);
16584 assert!(outcomes[1].output_tail.contains("command terminated:"));
16585 assert!(!outcomes[1].output_tail.contains("timed out"));
16586 assert_eq!(outcomes[2].code, Some(7));
16587 assert!(!outcomes[2].output_tail.contains("timed out"));
16588 assert_eq!(outcomes[3].code, Some(0));
16589 assert_eq!(outcomes[3].output_tail, "success");
16590 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16591 }
16592
16593 #[tokio::test]
16603 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16604 crate::run::pin_test_home();
16605
16606 let tmp = tempfile::tempdir().expect("tempdir");
16607 let repo = tmp.path().join("repo");
16608 std::fs::create_dir_all(&repo).unwrap();
16609 init_repo(&repo);
16610
16611 let mut config = Config::default();
16612 config.verify.gate = vec![
16613 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16614 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16615 .to_owned(),
16616 ];
16617
16618 let mut state = RunState::new(
16619 repo.clone(),
16620 "main".to_owned(),
16621 "deadbeef".to_owned(),
16622 "task".to_owned(),
16623 config,
16624 );
16625 let run_id = state.id.clone();
16626 state.candidates = vec![Candidate {
16627 index: 0,
16628 label: 'A',
16629 agent: "alpha".to_owned(),
16630 branch: "does-not-exist".to_owned(),
16631 worktree: repo.clone(),
16632 summary: String::new(),
16633 stat: String::new(),
16634 files: 0,
16635 commits: 0,
16636 empty: false,
16637 failed: None,
16638 verified_noop: None,
16639 duration_ms: 0,
16640 folded: false,
16641 }];
16642 state.tally = Some(Tally {
16643 first_choice: BTreeMap::from([('A', 1)]),
16644 borda: BTreeMap::new(),
16645 winner: 'A',
16646 rankings: 1,
16647 unanimous_initial: true,
16648 deliberated: false,
16649 changed_votes: 0,
16650 unanimous_final: true,
16651 tie_break: None,
16652 judges: 0,
16653 present: 0,
16654 quorum: 0,
16655 met_quorum: true,
16656 uncontested: Some("only candidate A produced a change".to_owned()),
16657 });
16658 state.reviews = vec![ReviewRound {
16659 round: 1,
16660 head: "deadbeef".to_owned(),
16661 verified_head: None,
16662 verified_at: None,
16663 reviews: Vec::new(),
16664 e2e: Vec::new(),
16665 fix: None,
16666 blocking: 0,
16667 answered: 0,
16668 expected: 0,
16669 clean: true,
16670 verify_retried: false,
16671 e2e_deferred: false,
16672 e2e_defer_reason: None,
16673 progressed: false,
16674 vote_split: false,
16675 reconsideration: Vec::new(),
16676 verdict: None,
16677 }];
16678
16679 let mut runner = Runner {
16680 state,
16681 roles: ResolvedRoles {
16682 implementers: Vec::new(),
16683 judges: Vec::new(),
16684 reviewers: Vec::new(),
16685 fixer: None,
16686 conductor: conductor(),
16687 implementer_roster: Vec::new(),
16688 judge_roster: Vec::new(),
16689 reviewer_roster: Vec::new(),
16690 },
16691 sem: Arc::new(Semaphore::new(1)),
16692 pause: Pause::new(),
16693 interrupt: Pause::new(),
16694 };
16695
16696 let started_marker = repo.join("started.marker");
16697 let release_marker = repo.join("release.marker");
16698 let poller = tokio::spawn(async move {
16699 for _ in 0..100 {
16704 if started_marker.exists()
16705 && let Ok(s) = crate::run::RunState::load(&run_id)
16706 && let Some(a) = s.active.get("gate")
16707 {
16708 std::fs::write(&release_marker, b"go").expect("release marker");
16709 return Some(a.clone());
16710 }
16711 tokio::time::sleep(Duration::from_millis(50)).await;
16712 }
16713 None
16714 });
16715
16716 runner.gate().await.expect("gate");
16717 let captured = poller.await.expect("poller task");
16718 let captured = captured.expect(
16719 "the poller never saw a `gate` task entry in run.json while the command was \
16720 still blocked on its own release marker",
16721 );
16722
16723 assert_eq!(captured.task.as_deref(), Some("gate"));
16724 assert_eq!(captured.node, "gate");
16725 assert_eq!(captured.index, Some(1));
16726 assert_eq!(captured.total, Some(1));
16727 assert!(
16728 captured
16729 .command
16730 .as_deref()
16731 .is_some_and(|c| c.contains("started.marker")),
16732 "{captured:?}"
16733 );
16734
16735 assert!(
16736 runner.state.active.is_empty(),
16737 "the entry must be cleared once the command actually finished: {:?}",
16738 runner.state.active
16739 );
16740 assert!(runner.state.gate_ran);
16741 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16742 }
16743
16744 #[tokio::test]
16747 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16748 use crate::verdict::{Finding, ReviewVote, Severity};
16749 crate::run::pin_test_home();
16750 let tmp = tempfile::tempdir().expect("tempdir");
16751 let repo = tmp.path().join("repo");
16752 std::fs::create_dir_all(&repo).unwrap();
16753 init_repo(&repo);
16754
16755 for (severity, vote, expect) in [
16756 (Severity::Major, ReviewVote::Reject, true),
16757 (Severity::Minor, ReviewVote::Reject, false),
16758 (Severity::Major, ReviewVote::Approve, false),
16759 ] {
16760 let mut round = review_round(false, 1, 1, 1, false, true);
16761 round.reviews = vec![ReviewRecord {
16762 reviewer: 1,
16763 agent: "alpha".to_owned(),
16764 summary: String::new(),
16765 findings: vec![Finding {
16766 id: "R1-1-1".to_owned(),
16767 severity,
16768 file: None,
16769 line: None,
16770 title: "t".to_owned(),
16771 detail: String::new(),
16772 }],
16773 vote: Some(vote),
16774 failed: None,
16775 duration_ms: 0,
16776 attempts: 0,
16777 }];
16778 let mut state = RunState::new(
16779 repo.clone(),
16780 "main".to_owned(),
16781 "deadbeef".to_owned(),
16782 "task".to_owned(),
16783 Config::default(),
16784 );
16785 state.reviews = vec![round];
16786 let mut runner = Runner {
16787 state,
16788 roles: ResolvedRoles {
16789 implementers: Vec::new(),
16790 judges: Vec::new(),
16791 reviewers: Vec::new(),
16792 fixer: None,
16793 conductor: conductor(),
16794 implementer_roster: Vec::new(),
16795 judge_roster: Vec::new(),
16796 reviewer_roster: Vec::new(),
16797 },
16798 sem: Arc::new(Semaphore::new(1)),
16799 pause: Pause::new(),
16800 interrupt: Pause::new(),
16801 };
16802 let shell = runner.state.config.shell();
16803 runner
16804 .stop_reviewing("round budget spent", &shell, &repo)
16805 .await
16806 .expect("stop_reviewing");
16807 assert_eq!(runner.state.status, RunStatus::Gating);
16808 assert_eq!(
16809 runner.state.contested_handoff.is_some(),
16810 expect,
16811 "{severity:?} + {vote:?}"
16812 );
16813 }
16814 }
16815
16816 #[tokio::test]
16829 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16830 crate::run::pin_test_home();
16831 let home = crate::run::home();
16832
16833 let tmp = tempfile::tempdir().expect("tempdir");
16834 let repo = tmp.path().join("repo");
16835 std::fs::create_dir_all(&repo).unwrap();
16836 init_repo(&repo);
16837 let head = crate::git::rev_parse(&repo, "HEAD")
16838 .await
16839 .expect("rev-parse");
16840 let cache_dir = tmp.path().join("target");
16843
16844 let mut config = Config::default();
16845 config.verify.e2e = vec![format!(
16846 "CARGO_TARGET_DIR='{}' test -f README.md",
16847 cache_dir.display()
16848 )];
16849 config.graph.review_rounds = 1;
16850 config.graph.timeout_verify = Some(2);
16853 config.graph.timeout_resource_wait = 2;
16854
16855 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16856 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16857 .expect("no io error acquiring directly")
16858 {
16859 crate::cache::AcquireOutcome::Acquired(g) => g,
16860 crate::cache::AcquireOutcome::Busy(b) => {
16861 panic!("expected the direct acquire to win the lease first: {b:?}")
16862 }
16863 };
16864
16865 let mut state = RunState::new(
16866 repo.clone(),
16867 "main".to_owned(),
16868 head.clone(),
16869 "task".to_owned(),
16870 config,
16871 );
16872 state.candidates = vec![Candidate {
16873 index: 0,
16874 label: 'A',
16875 agent: "alpha".to_owned(),
16876 branch: "does-not-exist".to_owned(),
16877 worktree: repo.clone(),
16878 summary: String::new(),
16879 stat: String::new(),
16880 files: 0,
16881 commits: 0,
16882 empty: false,
16883 failed: None,
16884 verified_noop: None,
16885 duration_ms: 0,
16886 folded: false,
16887 }];
16888 state.tally = Some(Tally {
16889 first_choice: BTreeMap::from([('A', 1)]),
16890 borda: BTreeMap::new(),
16891 winner: 'A',
16892 rankings: 1,
16893 unanimous_initial: true,
16894 deliberated: false,
16895 changed_votes: 0,
16896 unanimous_final: true,
16897 tie_break: None,
16898 judges: 0,
16899 present: 0,
16900 quorum: 0,
16901 met_quorum: true,
16902 uncontested: Some("only candidate A produced a change".to_owned()),
16903 });
16904 state.reviews = vec![ReviewRound {
16908 round: 1,
16909 head: head.clone(),
16910 verified_head: None,
16911 verified_at: None,
16912 reviews: Vec::new(),
16913 e2e: Vec::new(),
16914 fix: None,
16915 blocking: 1,
16916 answered: 1,
16917 expected: 1,
16918 clean: false,
16919 verify_retried: false,
16920 e2e_deferred: true,
16921 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16922 progressed: false,
16923 vote_split: false,
16924 reconsideration: Vec::new(),
16925 verdict: None,
16926 }];
16927
16928 let mut runner = Runner {
16929 state,
16930 roles: ResolvedRoles {
16931 implementers: Vec::new(),
16932 judges: Vec::new(),
16933 reviewers: Vec::new(),
16934 fixer: None,
16935 conductor: conductor(),
16936 implementer_roster: Vec::new(),
16937 judge_roster: Vec::new(),
16938 reviewer_roster: Vec::new(),
16939 },
16940 sem: Arc::new(Semaphore::new(1)),
16941 pause: Pause::new(),
16942 interrupt: Pause::new(),
16943 };
16944
16945 let shell = runner.state.config.shell();
16946 runner
16947 .stop_reviewing("round budget spent", &shell, &repo)
16948 .await
16949 .expect("stop_reviewing");
16950
16951 let last = runner.state.reviews.last().expect("round record");
16952 assert_eq!(
16953 last.e2e_status(),
16954 E2eStatus::ResourceBlocked,
16955 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16956 failed: {last:?}"
16957 );
16958 assert_eq!(
16959 last.verified_head.as_deref(),
16960 Some(head.as_str()),
16961 "which commit this attempt targeted is known even though nothing finished checking \
16962 it"
16963 );
16964 let first_attempt_at = last
16965 .verified_at
16966 .expect("when this attempt ran is known too");
16967 assert_ne!(
16968 runner.state.status,
16969 RunStatus::Blocked,
16970 "contention is evidence about the machine, not the patch — it must not settle the \
16971 run as blocked: {:?}",
16972 runner.state.status
16973 );
16974 assert!(
16975 !runner
16976 .state
16977 .events
16978 .iter()
16979 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16980 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16981 runner.state.events
16982 );
16983
16984 runner
16989 .stop_reviewing("round budget spent", &shell, &repo)
16990 .await
16991 .expect("stop_reviewing retry");
16992 assert_eq!(
16993 runner.state.reviews.len(),
16994 1,
16995 "no new round was started: {:?}",
16996 runner.state.reviews
16997 );
16998 let last = runner.state.reviews.last().expect("round record");
16999 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
17000 assert!(
17001 last.verified_at.expect("still known") > first_attempt_at,
17002 "a second reentry must be a fresh attempt, not a stale copy of the first"
17003 );
17004 assert_ne!(runner.state.status, RunStatus::Blocked);
17005
17006 held.release();
17007 }
17008
17009 #[cfg(unix)]
17010 #[tokio::test]
17011 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
17012 ask_test_home();
17013 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
17014 let tmp = tempfile::tempdir().unwrap();
17015 let repo = tmp.path().join("repo");
17016 std::fs::create_dir(&repo).unwrap();
17017 init_repo(&repo);
17018 let run_git = |args: &[&str]| {
17019 let out = std::process::Command::new("git")
17020 .args(args)
17021 .current_dir(&repo)
17022 .output()
17023 .unwrap();
17024 assert!(
17025 out.status.success(),
17026 "{args:?}: {}",
17027 String::from_utf8_lossy(&out.stderr)
17028 );
17029 };
17030 let origin = tmp.path().join("origin.git");
17031 run_git(&["init", "--bare", origin.to_str().unwrap()]);
17032 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17033 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17034 run_git(&["add", "spec.rs"]);
17035 run_git(&["commit", "-m", "initial spec"]);
17036 run_git(&["push", "origin", "main"]);
17037 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17038 run_git(&["checkout", "-b", "fix/review"]);
17039 std::fs::write(
17040 repo.join("check.rs"),
17041 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17042 )
17043 .unwrap();
17044 run_git(&["add", "check.rs"]);
17045 run_git(&["commit", "-m", "use spec"]);
17046 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17047 run_git(&["checkout", "main"]);
17048 std::fs::write(
17049 repo.join("spec.rs"),
17050 "struct AgentSpec { max_turn_secs: u64 }\n",
17051 )
17052 .unwrap();
17053 run_git(&["add", "spec.rs"]);
17054 run_git(&["commit", "-m", "extend spec"]);
17055 run_git(&["push", "origin", "main"]);
17056 run_git(&["checkout", "fix/review"]);
17057
17058 let mut runner = runner_at(RunStatus::Reviewing);
17059 runner.state.repo = repo.clone();
17060 runner.state.base_commit = base;
17061 runner.state.pr_head = Some("fix/review".into());
17062 runner.state.config.graph.review_rounds = max_rounds;
17063 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17064 runner.state.config.graph.sessions = false;
17065 runner.state.config.disk.min_free_bytes = 0;
17066 runner.state.config.verify.e2e =
17067 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17068 runner.state.candidates = vec![Candidate {
17069 index: 0,
17070 label: 'A',
17071 agent: "fixture".into(),
17072 branch: "fix/review".into(),
17073 worktree: repo.clone(),
17074 summary: String::new(),
17075 stat: String::new(),
17076 files: 1,
17077 commits: 1,
17078 empty: false,
17079 failed: None,
17080 verified_noop: None,
17081 duration_ms: 0,
17082 folded: false,
17083 }];
17084 runner.state.tally = Some(Tally {
17085 first_choice: BTreeMap::new(),
17086 borda: BTreeMap::new(),
17087 winner: 'A',
17088 rankings: 0,
17089 unanimous_initial: true,
17090 deliberated: false,
17091 changed_votes: 0,
17092 unanimous_final: true,
17093 tie_break: None,
17094 judges: 0,
17095 present: 0,
17096 quorum: 0,
17097 met_quorum: true,
17098 uncontested: Some("review-only".into()),
17099 });
17100 let mut last = review_round(false, 0, 1, 1, false, false);
17101 last.head = reviewed.clone();
17102 last.e2e_deferred = true;
17103 last.e2e[0].resource_blocked = true;
17104 last.e2e[0].output_tail = "waiting for the shared build cache".into();
17105 if max_rounds == 1 {
17106 last.e2e.clear();
17107 }
17108 runner.state.reviews = vec![last];
17109 let calls = tmp.path().join("fix-calls");
17110 let mut fixer = spec("repair");
17111 fixer
17112 .env
17113 .insert("FIX_CALLS".into(), calls.display().to_string());
17114 fixer.command = vec!["sh".into(), "-c".into(), r#"
17115 echo fix >> "$FIX_CALLS"
17116 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
17117 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
17118 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
17119 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
17120 "#.into()];
17121 let mut reviewer = spec("reviewer");
17122 reviewer.command = vec![
17123 "sh".into(),
17124 "-c".into(),
17125 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
17126 .into(),
17127 ];
17128 runner.roles.fixer = Some(vec![fixer]);
17129 runner.roles.reviewers = vec![reviewer];
17130
17131 runner.sync_to_base().await.unwrap();
17132 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
17133 assert_ne!(rebased, reviewed);
17134 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
17135 if reentry {
17136 let shell = runner.state.config.shell();
17139 runner
17140 .stop_unresolved_review("deferred verification", &shell, &repo)
17141 .await
17142 .unwrap();
17143 assert!(runner.catchup_repairable());
17144 assert!(!calls.exists());
17145 runner.state = RunState::load(&runner.state.id).unwrap();
17146 assert!(runner.catchup_repairable());
17147 }
17148 runner.review_loop().await.unwrap();
17149 let first = &runner.state.reviews[0];
17150 assert_eq!(first.head, reviewed);
17151 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
17152 assert!(
17153 first.verify_retried,
17154 "a reproduced build failure gets exactly one retry: {:?}",
17155 first.e2e
17156 );
17157 assert!(first.e2e[0].output_tail.contains("E0063"));
17158 assert!(!runner.catchup_repairable());
17159 if max_rounds == 2 {
17160 assert_eq!(
17161 runner.state.status,
17162 RunStatus::Gating,
17163 "{}",
17164 crate::report::run(&runner.state)
17165 );
17166 assert_eq!(runner.state.reviews.len(), 2);
17167 assert!(first.fix.as_ref().unwrap().committed);
17168 assert!(runner.state.reviews[1].clean);
17169 assert_eq!(
17170 runner
17171 .state
17172 .reviews
17173 .iter()
17174 .filter(|r| r.fix.is_some())
17175 .count(),
17176 1
17177 );
17178 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
17179 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
17180 } else {
17181 assert_eq!(runner.state.status, RunStatus::Blocked);
17182 assert!(first.fix.is_none());
17183 assert!(!calls.exists());
17184 let recovery = crate::recovery::assess(
17185 "prior",
17186 Some(&runner.state),
17187 &runner.landing_base(),
17188 crate::recovery::Relation::Diverged,
17189 );
17190 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
17191 let reasons = recovery.reasons.join("\n");
17192 assert!(
17193 reasons.contains(&reviewed) && reasons.contains(&rebased),
17194 "{reasons}"
17195 );
17196 assert!(
17197 reasons.contains("base sync to")
17198 && reasons.contains("additional review is required"),
17199 "{reasons}"
17200 );
17201 }
17202 }
17203 }
17204
17205 #[tokio::test]
17217 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
17218 crate::run::pin_test_home();
17219 let home = crate::run::home();
17220
17221 let tmp = tempfile::tempdir().expect("tempdir");
17222 let repo = tmp.path().join("repo");
17223 std::fs::create_dir_all(&repo).unwrap();
17224 init_repo(&repo);
17225 let head = crate::git::rev_parse(&repo, "HEAD")
17226 .await
17227 .expect("rev-parse");
17228 let cache_dir = tmp.path().join("target");
17229
17230 let mut config = Config::default();
17231 config.verify.e2e = vec![format!(
17232 "CARGO_TARGET_DIR='{}' test -f README.md",
17233 cache_dir.display()
17234 )];
17235 config.graph.review_rounds = 1;
17236 config.graph.timeout_verify = Some(2);
17237 config.graph.timeout_resource_wait = 2;
17238
17239 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17240 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17241 .expect("no io error acquiring directly")
17242 {
17243 crate::cache::AcquireOutcome::Acquired(g) => g,
17244 crate::cache::AcquireOutcome::Busy(b) => {
17245 panic!("expected the direct acquire to win the lease first: {b:?}")
17246 }
17247 };
17248
17249 let mut state = RunState::new(
17250 repo.clone(),
17251 "main".to_owned(),
17252 head.clone(),
17253 "task".to_owned(),
17254 config,
17255 );
17256 state.candidates = vec![Candidate {
17257 index: 0,
17258 label: 'A',
17259 agent: "alpha".to_owned(),
17260 branch: "does-not-exist".to_owned(),
17261 worktree: repo.clone(),
17262 summary: String::new(),
17263 stat: String::new(),
17264 files: 0,
17265 commits: 0,
17266 empty: false,
17267 failed: None,
17268 verified_noop: None,
17269 duration_ms: 0,
17270 folded: false,
17271 }];
17272 state.tally = Some(Tally {
17273 first_choice: BTreeMap::from([('A', 1)]),
17274 borda: BTreeMap::new(),
17275 winner: 'A',
17276 rankings: 1,
17277 unanimous_initial: true,
17278 deliberated: false,
17279 changed_votes: 0,
17280 unanimous_final: true,
17281 tie_break: None,
17282 judges: 0,
17283 present: 0,
17284 quorum: 0,
17285 met_quorum: true,
17286 uncontested: Some("only candidate A produced a change".to_owned()),
17287 });
17288 state.reviews = vec![ReviewRound {
17292 round: 1,
17293 head: head.clone(),
17294 verified_head: Some(head.clone()),
17295 verified_at: Some(jiff::Timestamp::now()),
17296 reviews: Vec::new(),
17297 e2e: vec![CommandOutcome {
17298 command: format!(
17299 "CARGO_TARGET_DIR='{}' test -f README.md",
17300 cache_dir.display()
17301 ),
17302 code: None,
17303 output_tail: "waiting for the shared build cache".to_owned(),
17304 duration_ms: 0,
17305 resource_blocked: true,
17306 }],
17307 fix: None,
17308 blocking: 1,
17309 answered: 1,
17310 expected: 1,
17311 clean: false,
17312 verify_retried: false,
17313 e2e_deferred: false,
17314 e2e_defer_reason: None,
17315 progressed: false,
17316 vote_split: false,
17317 reconsideration: Vec::new(),
17318 verdict: None,
17319 }];
17320
17321 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17322 let mut runner = Runner {
17323 state,
17324 roles: ResolvedRoles {
17325 implementers: Vec::new(),
17326 judges: Vec::new(),
17327 reviewers: Vec::new(),
17328 fixer: None,
17329 conductor: conductor(),
17330 implementer_roster: Vec::new(),
17331 judge_roster: Vec::new(),
17332 reviewer_roster: Vec::new(),
17333 },
17334 sem: Arc::new(Semaphore::new(1)),
17335 pause: Pause::new(),
17336 interrupt: Pause::new(),
17337 };
17338
17339 runner.review_loop().await.expect("review_loop");
17344
17345 assert_eq!(
17346 runner.state.reviews.len(),
17347 1,
17348 "no new round was started on top of the unresolved one: {:?}",
17349 runner.state.reviews
17350 );
17351 let last = &runner.state.reviews[0];
17352 assert_eq!(
17353 last.e2e_status(),
17354 E2eStatus::ResourceBlocked,
17355 "still contended: {last:?}"
17356 );
17357 assert!(
17358 last.verified_at.expect("still known") > first_attempt_at,
17359 "review_loop must have actually retried the check, not left it exactly as found"
17360 );
17361 assert_ne!(
17362 runner.state.status,
17363 RunStatus::Blocked,
17364 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17365 runner.state.status
17366 );
17367
17368 held.release();
17369 }
17370
17371 #[tokio::test]
17372 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17373 crate::run::pin_test_home();
17374 for status in [RunStatus::Blocked, RunStatus::Gating] {
17375 let mut runner = runner_at(status);
17376 runner.state.config.graph.land = true;
17377 runner.state.gate_ran = true;
17378 runner.state.merge = Some(MergeOutcome {
17379 mode: MergeMode::Pr,
17380 ok: false,
17381 detail: "the fixer produced no commit".into(),
17382 empty: false,
17383 });
17384 let url = "https://example.invalid/o/r/pull/1";
17385 runner.state.pr = Some(crate::run::PrRecord {
17386 url: url.into(),
17387 number: 1,
17388 state: "open".into(),
17389 checks: "red".into(),
17390 round: 1,
17391 rounds: 1,
17392 red_at_merge: Vec::new(),
17393 });
17394 runner.state.save().unwrap();
17395 runner.execute_graph().await.unwrap();
17399 assert_eq!(runner.state.status, RunStatus::Blocked);
17400 assert!(runner.state.land_sync_pending);
17401 assert!(runner.state.events.iter().any(|event|
17402 event.node == "land" && event.message == format!("watching {url}")));
17403 assert_eq!(
17404 runner.state.merge.as_ref().unwrap().detail,
17405 "the fixer produced no commit"
17406 );
17407 }
17408 }
17409
17410 #[tokio::test]
17411 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17412 crate::run::pin_test_home();
17413 let tmp = tempfile::tempdir().expect("tempdir");
17414 let repo = tmp.path().join("repo");
17415 std::fs::create_dir_all(&repo).unwrap();
17416 init_repo(&repo);
17417
17418 let mut config = Config::default();
17419 config.merge.mode = MergeMode::Pr;
17420 config.graph.land = true;
17421 config.graph.land_approval = false;
17422
17423 let mut state = RunState::new(
17424 repo.clone(),
17425 "main".to_owned(),
17426 "deadbeef".to_owned(),
17427 "task".to_owned(),
17428 config,
17429 );
17430 state.candidates = vec![Candidate {
17431 index: 0,
17432 label: 'A',
17433 agent: "alpha".to_owned(),
17434 branch: "does-not-exist".to_owned(),
17435 worktree: repo.clone(),
17436 summary: String::new(),
17437 stat: String::new(),
17438 files: 0,
17439 commits: 0,
17440 empty: false,
17441 failed: None,
17442 verified_noop: None,
17443 duration_ms: 0,
17444 folded: false,
17445 }];
17446 state.tally = Some(Tally {
17447 first_choice: BTreeMap::from([('A', 1)]),
17448 borda: BTreeMap::new(),
17449 winner: 'A',
17450 rankings: 1,
17451 unanimous_initial: true,
17452 deliberated: false,
17453 changed_votes: 0,
17454 unanimous_final: true,
17455 tie_break: None,
17456 judges: 0,
17457 present: 0,
17458 quorum: 0,
17459 met_quorum: true,
17460 uncontested: Some("only candidate A produced a change".to_owned()),
17461 });
17462 state.reviews = vec![ReviewRound {
17463 round: 1,
17464 head: "deadbeef".to_owned(),
17465 verified_head: None,
17466 verified_at: None,
17467 reviews: Vec::new(),
17468 e2e: Vec::new(),
17469 fix: None,
17470 blocking: 0,
17471 answered: 0,
17472 expected: 0,
17473 clean: true,
17474 verify_retried: false,
17475 e2e_deferred: false,
17476 e2e_defer_reason: None,
17477 progressed: false,
17478 vote_split: false,
17479 reconsideration: Vec::new(),
17480 verdict: None,
17481 }];
17482 state.gate = vec![CommandOutcome {
17483 command: "test".to_owned(),
17484 code: Some(0),
17485 output_tail: String::new(),
17486 duration_ms: 0,
17487 resource_blocked: false,
17488 }];
17489 state.gate_ran = true;
17490 state.status = RunStatus::Landing;
17494 state.merge = Some(MergeOutcome {
17495 mode: MergeMode::Pr,
17496 ok: true,
17497 detail: "https://example.invalid/x/y/pull/1".to_owned(),
17498 empty: false,
17499 });
17500
17501 ask_test_home();
17505 state.save().expect("save landing run");
17508 let store = ask::Questions::open();
17509 let q = ask_open_question(&store, &state.id);
17510
17511 let mut runner = Runner {
17512 state,
17513 roles: ResolvedRoles {
17514 implementers: Vec::new(),
17515 judges: Vec::new(),
17516 reviewers: Vec::new(),
17517 fixer: None,
17518 conductor: conductor(),
17519 implementer_roster: Vec::new(),
17520 judge_roster: Vec::new(),
17521 reviewer_roster: Vec::new(),
17522 },
17523 sem: Arc::new(Semaphore::new(1)),
17524 pause: Pause::new(),
17525 interrupt: Pause::new(),
17526 };
17527
17528 runner.execute().await.expect("execute");
17533
17534 assert_eq!(
17535 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17536 Some("https://example.invalid/x/y/pull/1"),
17537 "reentry must not push again or open a second pull request over the \
17538 one `land` is already watching"
17539 );
17540 assert_ne!(
17541 runner.state.status,
17542 RunStatus::Landing,
17543 "land could not actually reach the fake pull request, so it must \
17544 have given up rather than left the run silently parked forever"
17545 );
17546 assert_eq!(runner.state.status, RunStatus::Blocked);
17550 assert!(
17551 store.get(&q.id).unwrap().status.open(),
17552 "Blocked is still alive; settle_questions must have been a no-op here"
17553 );
17554 }
17555
17556 fn state_with_round(round: ReviewRound) -> RunState {
17557 let mut s = RunState::new(
17558 PathBuf::from("/repo"),
17559 "main".to_owned(),
17560 "abc1234".to_owned(),
17561 "add retries".to_owned(),
17562 Config::default(),
17563 );
17564 s.reviews = vec![round];
17565 s
17566 }
17567
17568 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17569 crate::verdict::Finding {
17570 id: id.to_owned(),
17571 severity,
17572 file: None,
17573 line: None,
17574 title: title.to_owned(),
17575 detail: String::new(),
17576 }
17577 }
17578
17579 #[test]
17580 fn pr_body_names_open_findings_and_declined_ones() {
17581 let round = ReviewRound {
17582 round: 2,
17583 head: "deadbee".to_owned(),
17584 verified_head: None,
17585 verified_at: None,
17586 reviews: vec![ReviewRecord {
17587 attempts: 0,
17588 reviewer: 1,
17589 agent: "alpha".to_owned(),
17590 summary: String::new(),
17591 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17592 vote: None,
17593 failed: None,
17594 duration_ms: 0,
17595 }],
17596 e2e: vec![CommandOutcome {
17597 command: "cargo test".to_owned(),
17598 code: Some(0),
17599 output_tail: String::new(),
17600 duration_ms: 0,
17601 resource_blocked: false,
17602 }],
17603 verify_retried: false,
17604 e2e_deferred: false,
17605 e2e_defer_reason: None,
17606 fix: Some(FixRecord {
17607 agent: "alpha".to_owned(),
17608 addressed: Vec::new(),
17609 rejected: vec![crate::verdict::Rejection {
17610 id: "R1-1-1".to_owned(),
17611 why: "not reachable from any caller".to_owned(),
17612 }],
17613 notes: String::new(),
17614 committed: true,
17615 failed: None,
17616 duration_ms: 0,
17617 continuation: None,
17618 }),
17619 blocking: 0,
17620 answered: 1,
17621 expected: 1,
17622 clean: false,
17623 progressed: true,
17624 vote_split: false,
17625 reconsideration: Vec::new(),
17626 verdict: None,
17627 };
17628 let state = state_with_round(round);
17629 let body = pr_message(&state, 'A').body;
17630
17631 assert!(body.contains("add retries"), "the task must still be there");
17632 assert!(body.contains("R2-1-1"), "{body}");
17633 assert!(!body.contains("unused import"), "{body}");
17634 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17635 assert!(
17636 !body.contains("not reachable from any caller"),
17637 "the reason it was declined: {body}"
17638 );
17639 }
17640
17641 #[test]
17642 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17643 let round = ReviewRound {
17644 round: 1,
17645 head: "deadbee".to_owned(),
17646 verified_head: None,
17647 verified_at: None,
17648 reviews: vec![ReviewRecord {
17649 attempts: 0,
17650 reviewer: 1,
17651 agent: "alpha".to_owned(),
17652 summary: String::new(),
17653 findings: Vec::new(),
17654 vote: None,
17655 failed: None,
17656 duration_ms: 0,
17657 }],
17658 e2e: Vec::new(),
17659 verify_retried: false,
17660 e2e_deferred: false,
17661 e2e_defer_reason: None,
17662 fix: None,
17663 blocking: 0,
17664 answered: 1,
17665 expected: 1,
17666 clean: true,
17667 progressed: false,
17668 vote_split: false,
17669 reconsideration: Vec::new(),
17670 verdict: None,
17671 };
17672 let state = state_with_round(round);
17673 let body = pr_message(&state, 'A').body;
17674 assert!(!body.contains("Open review findings"), "{body}");
17675 assert!(!body.contains("Declined"), "{body}");
17676 }
17677
17678 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17679 let mut state = RunState::new(
17680 PathBuf::from("/repo"),
17681 "main".to_owned(),
17682 "abc1234".to_owned(),
17683 instruction.to_owned(),
17684 Config::default(),
17685 );
17686 state.candidates.push(Candidate {
17687 index: 0,
17688 label: 'A',
17689 agent: "alpha".to_owned(),
17690 branch: "magi/x/A".to_owned(),
17691 worktree: PathBuf::from("/wt"),
17692 summary: summary.to_owned(),
17693 stat: String::new(),
17694 files: 1,
17695 commits: 1,
17696 empty: false,
17697 failed: None,
17698 verified_noop: None,
17699 folded: false,
17700 duration_ms: 0,
17701 });
17702 state
17703 }
17704
17705 fn review_state(subjects: &[&str]) -> RunState {
17706 let mut state = state_with_summary(
17707 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17708 "",
17709 );
17710 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17711 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17712 state
17713 }
17714
17715 #[tokio::test]
17718 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17719 ask_test_home();
17720 let tmp = tempfile::tempdir().unwrap();
17721 let repo = tmp.path().join("repo");
17722 std::fs::create_dir_all(&repo).unwrap();
17723 init_repo(&repo);
17724 let origin = tmp.path().join("origin.git");
17725 let g = |dir: &Path, args: &[&str]| {
17726 let out = std::process::Command::new("git")
17727 .args(args)
17728 .current_dir(dir)
17729 .quiet()
17730 .output()
17731 .expect("spawn git");
17732 assert!(
17733 out.status.success(),
17734 "git {args:?}: {}",
17735 String::from_utf8_lossy(&out.stderr)
17736 );
17737 };
17738 g(
17739 tmp.path(),
17740 &[
17741 "clone",
17742 "--bare",
17743 "-q",
17744 repo.to_str().unwrap(),
17745 origin.to_str().unwrap(),
17746 ],
17747 );
17748 g(
17749 &repo,
17750 &["remote", "add", "origin", origin.to_str().unwrap()],
17751 );
17752 let c1 = git::rev_parse(&repo, "main").await.unwrap();
17753
17754 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17756 g(&repo, &["add", "-A"]);
17757 g(
17758 &repo,
17759 &["commit", "-q", "-m", "chore(deps): update a crate"],
17760 );
17761 g(&repo, &["push", "-q", "origin", "main"]);
17762 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17763 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17764 g(&repo, &["add", "-A"]);
17765 g(
17766 &repo,
17767 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17768 );
17769 g(&repo, &["checkout", "-q", "main"]);
17770
17771 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17772 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17773 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17775 assert_eq!(fallback, c1);
17776
17777 let mut state = review_state(&[
17778 "chore(deps): update a crate",
17779 "fix(daemon): apply a chosen action",
17780 ]);
17781 state.repo = repo.clone();
17782 state.base_branch = "main".to_owned();
17783 state.base_commit = c1;
17784 refresh_reviewed_commits(&mut state, "feat/own").await;
17785 assert_eq!(
17786 state.reviewed_commits,
17787 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17788 );
17789 assert_eq!(
17790 review_title(&state).as_deref(),
17791 Some("fix(daemon): apply a chosen action")
17792 );
17793 assert_eq!(
17794 leaked_subjects(&state, "feat/own").await,
17795 Some(vec!["chore(deps): update a crate".to_owned()])
17796 );
17797 state.config.merge.remote = "nowhere".to_owned();
17800 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17801 }
17802
17803 struct SummaryFixture {
17807 _tmp: tempfile::TempDir,
17808 repo: PathBuf,
17809 reply: PathBuf,
17810 runner: Runner,
17811 }
17812
17813 fn summary_fixture() -> SummaryFixture {
17814 ask_test_home();
17815 let tmp = tempfile::tempdir().unwrap();
17816 let repo = tmp.path().join("repo");
17817 std::fs::create_dir_all(&repo).unwrap();
17818 init_repo(&repo);
17819 let origin = tmp.path().join("origin.git");
17820 let g = |args: &[&str]| {
17821 let out = std::process::Command::new("git")
17822 .args(args)
17823 .current_dir(&repo)
17824 .quiet()
17825 .output()
17826 .expect("spawn git");
17827 assert!(
17828 out.status.success(),
17829 "git {args:?}: {}",
17830 String::from_utf8_lossy(&out.stderr)
17831 );
17832 };
17833 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17834 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17835 g(&["push", "-q", "origin", "main"]);
17836 g(&["checkout", "-q", "-b", "feat/own"]);
17837 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17838 g(&["add", "-A"]);
17839 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17840 let reply = tmp.path().join("reply.txt");
17841 std::fs::write(
17842 &reply,
17843 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17844 )
17845 .unwrap();
17846 let mut runner = runner_at(RunStatus::Gating);
17847 let mut state = review_state(&["feat(own): add the own file"]);
17848 state.repo = repo.clone();
17849 state.base_branch = "main".to_owned();
17850 state.candidates[0].branch = "feat/own".to_owned();
17851 state.candidates[0].worktree = repo.clone();
17852 let mut author = spec("alpha");
17853 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17854 author
17855 .env
17856 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17857 state.config.agents = vec![author];
17858 runner.state = state;
17859 SummaryFixture {
17860 _tmp: tmp,
17861 repo,
17862 reply,
17863 runner,
17864 }
17865 }
17866
17867 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17868 let winner = f.runner.state.candidates[0].clone();
17869 f.runner
17870 .guarded_pr_message(&winner, None, true)
17871 .await
17872 .unwrap()
17873 .expect("an English description never parks")
17874 }
17875
17876 fn clean_round_with_minor_findings() -> ReviewRound {
17877 let mut round = passing_round(2, None);
17878 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17879 round.reviews = vec![ReviewRecord {
17880 attempts: 0,
17881 reviewer: 1,
17882 agent: "alpha".into(),
17883 summary: "内部レビュー".into(),
17884 findings: vec![
17885 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17886 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17887 ],
17888 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17889 failed: None,
17890 duration_ms: 0,
17891 }];
17892 round
17893 }
17894
17895 #[tokio::test]
17896 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17897 for review_only in [false, true] {
17898 let mut f = summary_fixture();
17899 if !review_only {
17900 f.runner.state.candidates[0].agent = "alpha".into();
17901 }
17902 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17903 assert!(f.runner.state.reviews[0].clean);
17904 assert!(f.runner.state.reviews[0].fix.is_none());
17905 assert!(f.runner.state.open_findings().is_empty());
17906 assert_eq!(
17907 crate::pr_outcome::required_finding_ids(&f.runner.state),
17908 ["R2-1-1", "R2-1-2"]
17909 );
17910 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17911 let evidence = crate::pr_outcome::summary_evidence(
17912 &f.runner.state,
17913 crate::pr_outcome::FinalHead::At(&head),
17914 );
17915 assert!(
17916 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17917 );
17918 let omitted = posted(&mut f).await;
17920 assert!(omitted.body.contains("No description"));
17921 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17922 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17923 assert!(f.runner.state.change_summary.is_none());
17924 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17925 let described = posted(&mut f).await;
17926 assert!(described.body.contains("improve documentation for callers"));
17927 assert!(described.body.contains("remaining edge case"));
17928 assert!(described.body.contains("## Open review findings"));
17929 assert!(!described.body.contains("未修正"));
17930 }
17931 }
17932
17933 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17934 let mut f = summary_fixture();
17935 if !review_only {
17936 f.runner.state.candidates[0].agent = "alpha".into();
17937 }
17938 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17939 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17940 fixed
17941 .fix
17942 .as_mut()
17943 .unwrap()
17944 .rejected
17945 .push(crate::verdict::Rejection {
17946 id: "R1-1-1".into(),
17947 why: "呼び出されないコードのため辞退".into(),
17948 });
17949 let mut clean = clean_round_with_minor_findings();
17950 clean.verified_head = Some(head.clone());
17951 f.runner.state.reviews = vec![fixed, clean];
17952 ok_gate(&mut f.runner.state, Some(&head));
17953 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17954 f.runner.state.config.agents.push(scripted("beta", reply));
17955 f.runner.state.config.roles.language_judge =
17956 Some(crate::config::AgentChoice::One("beta".into()));
17957 f
17958 }
17959
17960 #[tokio::test]
17961 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17962 for review_only in [false, true] {
17963 let mut f = rewrite_fixture_with_findings(review_only,
17964 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17965 let m = posted(&mut f).await;
17966 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17967 let expected =
17968 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17969 assert!(m.body.contains(&expected));
17970 assert!(m.body.contains("## Review rounds"));
17971 assert!(
17972 m.body
17973 .contains("Gate (`verify.gate`): passed on the final commit")
17974 );
17975 assert!(
17976 m.body
17977 .contains("Tests (`verify.e2e`): passed on the final commit")
17978 );
17979 let c = f.runner.state.change_summary.as_ref().unwrap();
17980 assert!(
17981 m.body
17982 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17983 );
17984 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17985 assert!(crate::pr_outcome::covers_required_findings(
17986 &f.runner.state,
17987 &m.body
17988 ));
17989 assert!(m.body.contains("documentation remains incomplete"));
17990 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17991 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17992 assert!(f.runner.state.github_text.is_none());
17993 }
17994 }
17995
17996 #[tokio::test]
17997 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17998 for review_only in [false, true] {
17999 for reply in [
18000 "TITLE: fix: public description\nA short English description.",
18001 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
18002 ] {
18003 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
18004 let winner = f.runner.state.candidates[0].clone();
18005 let posted = f
18006 .runner
18007 .guarded_pr_message(&winner, None, true)
18008 .await
18009 .unwrap();
18010 assert!(posted.is_none());
18011 assert!(f.runner.state.parked);
18012 let withheld = f.runner.state.github_text.as_ref().unwrap();
18013 assert!(
18014 withheld.rewrite.is_none(),
18015 "an incomplete rewrite must not be selectable"
18016 );
18017 let q = ask::Questions::open()
18018 .get(withheld.question.as_ref().unwrap())
18019 .unwrap();
18020 assert!(
18021 !q.choices
18022 .contains(&crate::github_text::USE_REWRITE.to_owned())
18023 );
18024 assert!(
18025 q.detail.contains("R1-1-1")
18026 && q.detail.contains("R2-1-1")
18027 && q.detail.contains("R2-1-2")
18028 );
18029 }
18030 }
18031 }
18032
18033 #[tokio::test]
18034 async fn japanese_reports_stay_local_in_both_public_run_modes() {
18035 for review_only in [false, true] {
18036 let mut f = summary_fixture();
18037 if !review_only {
18038 f.runner.state.candidates[0].agent = "alpha".into();
18039 }
18040 f.runner.state.instruction = "内部タスクの説明".into();
18041 f.runner.state.candidates[0].summary = "内部の候補報告".into();
18042 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18043 f.runner.state.reviews = vec![
18044 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18045 round_with_notes(2, Some("二回目の内部修正報告です。")),
18046 passing_round(3, Some(&head)),
18047 ];
18048 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18049 ok_gate(&mut f.runner.state, Some(&head));
18050 let captured = f.repo.join("public-prompt.txt");
18051 let author = &mut f.runner.state.config.agents[0];
18052 author.command = vec![
18053 "sh".into(),
18054 "-c".into(),
18055 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18056 ];
18057 author.env.insert(
18058 "PROMPT_FILE".into(),
18059 captured.to_string_lossy().into_owned(),
18060 );
18061 let m = posted(&mut f).await;
18062 assert!(m.body.contains("Adds `own.txt`"));
18063 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18064 assert!(m.body.contains("R1-1-1"));
18065 assert!(
18066 m.body
18067 .contains("Gate (`verify.gate`): passed on the final commit")
18068 );
18069 assert!(
18070 m.body
18071 .contains("Tests (`verify.e2e`): passed on the final commit")
18072 );
18073 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18074 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18075 let prompt = std::fs::read_to_string(captured).unwrap();
18076 assert!(prompt.contains(&format!(
18077 "git diff {} {head}",
18078 f.runner.state.change_summary.as_ref().unwrap().base
18079 )));
18080 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18081 assert_eq!(
18082 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18083 "二回目の内部修正報告です。"
18084 );
18085 }
18086 }
18087
18088 #[tokio::test]
18089 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18090 for review_only in [false, true] {
18091 let mut f = summary_fixture();
18092 if !review_only {
18093 f.runner.state.candidates[0].agent = "alpha".into();
18094 }
18095 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18096 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18097 std::fs::write(&f.reply, "no usable description").unwrap();
18098 let failed = posted(&mut f).await;
18099 assert!(
18100 failed
18101 .body
18102 .contains("No description of what this change does could be generated")
18103 );
18104 assert!(failed.body.is_ascii());
18105 assert!(f.runner.state.github_text.is_none());
18106 assert!(f.runner.state.change_summary.is_none());
18107 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18108 let evidence = crate::pr_outcome::summary_evidence(
18109 &f.runner.state,
18110 crate::pr_outcome::FinalHead::At(&head),
18111 );
18112 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18113 assert!(
18114 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
18115 .exists()
18116 );
18117 f.runner.state =
18118 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
18119 std::fs::write(
18120 &f.reply,
18121 "TITLE: feat(own): add the own file\nAdds the committed own file.",
18122 )
18123 .unwrap();
18124 let retried = posted(&mut f).await;
18125 assert!(retried.body.contains("Adds the committed own file."));
18126 assert_eq!(
18127 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
18128 "日本語の修正報告"
18129 );
18130 }
18131 }
18132
18133 #[tokio::test]
18134 async fn same_head_verification_and_findings_invalidate_public_prose() {
18135 let mut f = summary_fixture();
18136 posted(&mut f).await;
18137 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18138 let original = f
18139 .runner
18140 .state
18141 .change_summary
18142 .as_ref()
18143 .unwrap()
18144 .evidence
18145 .clone();
18146 ok_gate(&mut f.runner.state, Some(&head));
18147 std::fs::write(
18148 &f.reply,
18149 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
18150 )
18151 .unwrap();
18152 let verified = posted(&mut f).await;
18153 assert!(verified.body.contains("now verified"));
18154 assert_ne!(
18155 f.runner.state.change_summary.as_ref().unwrap().evidence,
18156 original
18157 );
18158 let mut round = round_with_notes(1, Some("内部報告"));
18159 round.reviews.push(ReviewRecord {
18160 attempts: 0,
18161 reviewer: 1,
18162 agent: "alpha".into(),
18163 summary: "内部レビュー".into(),
18164 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
18165 vote: None,
18166 failed: None,
18167 duration_ms: 0,
18168 });
18169 round
18170 .fix
18171 .as_mut()
18172 .unwrap()
18173 .rejected
18174 .push(crate::verdict::Rejection {
18175 id: "R1-1-1".into(),
18176 why: "呼び出し元から到達できない".into(),
18177 });
18178 f.runner.state.reviews = vec![round];
18179 let omitted = posted(&mut f).await;
18181 assert!(omitted.body.contains("No description"));
18182 assert!(omitted.body.contains("R1-1-1"));
18183 assert!(!omitted.body.contains("到達"));
18184 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
18185 let described = posted(&mut f).await;
18186 assert!(described.body.contains("unreachable from callers"));
18187 assert!(described.body.contains("R1-1-1"));
18188 assert!(!described.body.contains("到達"));
18189 let evidence = f
18190 .runner
18191 .state
18192 .change_summary
18193 .as_ref()
18194 .unwrap()
18195 .evidence
18196 .clone();
18197 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
18198 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
18199 assert!(posted(&mut f).await.body.contains("rationale updated"));
18200 assert_ne!(
18201 f.runner.state.change_summary.as_ref().unwrap().evidence,
18202 evidence
18203 );
18204 f.runner
18206 .state
18207 .change_summary
18208 .as_mut()
18209 .unwrap()
18210 .evidence
18211 .clear();
18212 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
18213 assert!(posted(&mut f).await.body.contains("current specification"));
18214 }
18215
18216 #[tokio::test]
18217 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
18218 let mut f = summary_fixture();
18219 let m = posted(&mut f).await;
18220 assert_eq!(m.title, "feat(own): add the own file");
18221 assert!(
18222 m.body
18223 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
18224 "{}",
18225 m.body
18226 );
18227 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
18228 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
18229 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18230 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18231 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
18232 let kept = f.runner.state.change_summary.as_ref().unwrap();
18233 assert_eq!(
18234 (kept.head.as_str(), kept.base.as_str()),
18235 (head.as_str(), base.as_str())
18236 );
18237 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
18238 }
18239
18240 #[tokio::test]
18241 async fn review_only_summary_receives_the_configured_github_language() {
18242 let mut f = summary_fixture();
18243 f.runner.state.config.graph.github_language = "ja".into();
18244 let captured = f.repo.join("summary-prompt.txt");
18245 let author = &mut f.runner.state.config.agents[0];
18246 author.command = vec![
18247 "sh".into(),
18248 "-c".into(),
18249 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18250 ];
18251 author.env.insert(
18252 "PROMPT_FILE".into(),
18253 captured.to_string_lossy().into_owned(),
18254 );
18255 std::fs::write(
18256 &f.reply,
18257 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18258 )
18259 .unwrap();
18260 let m = posted(&mut f).await;
18261 let prompt = std::fs::read_to_string(captured).unwrap();
18262 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18263 assert!(!prompt.contains("in English"));
18264 assert_eq!(m.title, "feat(own): ファイルを追加");
18265 assert!(m.body.contains("own.txt を追加します。"));
18266 }
18267
18268 #[tokio::test]
18269 async fn review_fixes_come_after_the_whole_change_summary() {
18270 let mut f = summary_fixture();
18271 f.runner.state.reviews = vec![
18272 round_with_notes(1, Some("handled the PATH case")),
18273 round_with_notes(2, None),
18274 ];
18275 let m = posted(&mut f).await;
18276 let summary = m.body.find("## Summary").expect("summary");
18277 let fixes = m.body.find("## Review rounds").expect("review rounds");
18278 assert!(summary < fixes, "{}", m.body);
18279 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18280 assert!(m.body.contains("## Final verification"), "{}", m.body);
18281 }
18282
18283 #[tokio::test]
18284 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18285 let mut f = summary_fixture();
18286 let first = posted(&mut f).await;
18287 let first_head = f.runner.state.change_summary.clone().unwrap().head;
18288 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18290 let again = posted(&mut f).await;
18291 assert_eq!((again.title, again.body), (first.title, first.body));
18292
18293 let g = |args: &[&str]| {
18294 let out = std::process::Command::new("git")
18295 .args(args)
18296 .current_dir(&f.repo)
18297 .quiet()
18298 .output()
18299 .expect("spawn git");
18300 assert!(out.status.success(), "git {args:?}");
18301 };
18302 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18303 g(&["add", "-A"]);
18304 g(&["commit", "-q", "-m", "fix(own): handle more"]);
18305 let second = posted(&mut f).await;
18306 let kept = f.runner.state.change_summary.clone().unwrap();
18307 assert_ne!(kept.head, first_head);
18308 assert!(second.body.contains("Not read yet."), "{}", second.body);
18309 assert!(!second.body.contains("so the first summary is read"));
18310 assert_eq!(second.title, "feat(own): other");
18311 }
18312
18313 #[tokio::test]
18314 async fn an_unavailable_summary_is_stated_and_never_invented() {
18315 let mut f = summary_fixture();
18316 std::fs::write(&f.reply, "no title line here").unwrap();
18317 let m = posted(&mut f).await;
18318 assert!(
18319 m.body
18320 .contains("No description of what this change does could be generated"),
18321 "{}",
18322 m.body
18323 );
18324 assert!(m.body.contains("internal reports remain in local records"));
18325 assert!(f.runner.state.change_summary.is_none());
18326 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18327 let winner = f.runner.state.candidates[0].clone();
18329 let preview = f
18330 .runner
18331 .guarded_pr_message(&winner, None, false)
18332 .await
18333 .unwrap()
18334 .unwrap();
18335 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18336 }
18337
18338 #[test]
18339 fn pr_message_review_single_commit_uses_its_subject() {
18340 let state = review_state(&["feat(nats): per-role user"]);
18341 let m = pr_message(&state, 'A');
18342 assert_eq!(m.title, "feat(nats): per-role user");
18343 assert!(!m.body.contains("Review the work already"), "{}", m.body);
18344 assert!(m.body.contains("## Commits under review"), "{}", m.body);
18345 }
18346
18347 #[test]
18348 fn pr_message_review_multi_commit_takes_the_oldest() {
18349 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18350 let m = pr_message(&state, 'A');
18351 assert_eq!(m.title, "feat: the change");
18352 for s in ["feat: the change", "fix: typo", "fix: again"] {
18353 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18354 }
18355 }
18356
18357 #[test]
18358 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18359 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18360 let mut state = review_state(&[first, "fix: later fixup"]);
18361 state.candidates[0].branch = "機能/ログイン".to_owned();
18362 let m = pr_message(&state, 'A');
18363 assert!(
18364 m.title.starts_with("chore: land candidate A of run"),
18365 "{}",
18366 m.title
18367 );
18368 }
18369 }
18370
18371 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18372 BranchFacts {
18373 commits: commits
18374 .iter()
18375 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18376 .collect(),
18377 stat: stat.to_owned(),
18378 }
18379 }
18380
18381 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18382 let mut r = review_round(true, 0, 1, 1, true, true);
18383 r.round = round;
18384 r.fix = notes.map(|n| FixRecord {
18385 agent: "fixer".to_owned(),
18386 addressed: Vec::new(),
18387 rejected: Vec::new(),
18388 notes: n.to_owned(),
18389 committed: true,
18390 failed: None,
18391 duration_ms: 0,
18392 continuation: None,
18393 });
18394 r
18395 }
18396
18397 #[test]
18398 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18399 let state = review_state(&["ignored"]);
18400 let f = facts(
18401 &[
18402 (
18403 "fix(login): resolve PATH on macOS",
18404 "Login shells skip rc files.",
18405 ),
18406 ("fix: address review", ""),
18407 ],
18408 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18409 );
18410 let m = pr_message_with(&state, 'A', Some(&f));
18411 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18412 assert!(
18413 m.body
18414 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
18415 "{}",
18416 m.body
18417 );
18418 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18419 assert!(m.body.contains("## Diff stat"), "{}", m.body);
18420 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18421 for banned in [
18422 "Review the work already",
18423 "no task statement",
18424 "Original task",
18425 ] {
18426 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18427 }
18428 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18429 }
18430
18431 #[test]
18432 fn pr_message_review_truncates_a_huge_first_commit_body() {
18433 let state = review_state(&["ignored"]);
18434 let f = facts(
18435 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18436 "s",
18437 );
18438 let m = pr_message_with(&state, 'A', Some(&f));
18439 assert!(m.body.len() < 30_000, "{}", m.body.len());
18440 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18441 assert!(
18442 m.body.contains("- ... 1 more commit(s)"),
18443 "{}",
18444 m.body.len()
18445 );
18446 assert!(m.body.ends_with("magi:candidate-a\n"));
18447 }
18448
18449 #[test]
18450 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18451 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18452 assert_eq!(m.title, "feat: x");
18453 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18454 assert!(!m.body.contains("Diff stat"), "{}", m.body);
18455 assert!(!m.body.contains("no task statement"), "{}", m.body);
18456 }
18457
18458 #[test]
18459 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18460 let mut state = review_state(&["日本語の件名"]);
18461 state.candidates[0].branch = "fix/macos-login-path".to_owned();
18462 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18463 state.candidates[0].branch = "機能/ログイン".to_owned();
18464 assert!(
18465 pr_message(&state, 'A')
18466 .title
18467 .starts_with("chore: land candidate A")
18468 );
18469 }
18470
18471 #[test]
18472 fn pr_message_review_fixes_survive_a_clean_final_round() {
18473 let mut state = review_state(&["feat: x"]);
18474 state.reviews = vec![
18475 round_with_notes(1, Some("handled the PATH case")),
18476 round_with_notes(2, None),
18477 ];
18478 let body = pr_message(&state, 'A').body;
18479 assert!(
18480 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18481 "{body}"
18482 );
18483 assert!(body.contains("### Round 1"), "{body}");
18484
18485 state.reviews = vec![
18486 round_with_notes(1, Some("first fix")),
18487 round_with_notes(2, Some("")),
18488 round_with_notes(3, Some("second fix")),
18489 round_with_notes(4, None),
18490 ];
18491 let body = pr_message(&state, 'A').body;
18492 assert!(
18493 body.contains("### Round 1") && !body.contains("first fix"),
18494 "{body}"
18495 );
18496 assert!(
18497 body.contains("### Round 3") && !body.contains("second fix"),
18498 "{body}"
18499 );
18500 assert!(body.contains("### Round 2"), "{body}");
18501 }
18502
18503 const FINAL: &str = "abcdef1234567890";
18504
18505 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18506 let mut r = round_with_notes(round, None);
18507 r.verified_head = verified.map(str::to_owned);
18508 r
18509 }
18510
18511 fn ok_gate(state: &mut RunState, head: Option<&str>) {
18512 state.gate = vec![CommandOutcome {
18513 command: "gate".to_owned(),
18514 code: Some(0),
18515 output_tail: String::new(),
18516 duration_ms: 0,
18517 resource_blocked: false,
18518 }];
18519 state.gate_ran = true;
18520 state.gate_head = head.map(str::to_owned);
18521 }
18522
18523 fn body_at(state: &RunState) -> String {
18524 pr_message_raw(
18525 state,
18526 'A',
18527 None,
18528 crate::pr_outcome::FinalHead::At(FINAL),
18529 ChangeNote::NotAsked,
18530 )
18531 .body
18532 }
18533
18534 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18535
18536 #[test]
18537 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18538 let mut state = review_state(&["feat: x"]);
18539 state.reviews = vec![
18540 round_with_notes(1, Some(SANDBOXED)),
18541 passing_round(2, Some(FINAL)),
18542 ];
18543 ok_gate(&mut state, Some(FINAL));
18544 let body = body_at(&state);
18545 assert!(!body.contains("Handled the PATH case."), "{body}");
18546 assert!(!body.contains("could not commit"), "{body}");
18547 assert!(!body.contains("could not run"), "{body}");
18548 assert!(
18549 body.contains("The fix is committed on the branch"),
18550 "{body}"
18551 );
18552 assert!(!body.contains("sandbox"), "{body}");
18553 assert!(
18554 body.contains("Tests (`verify.e2e`): passed on the final commit"),
18555 "{body}"
18556 );
18557 assert!(
18558 body.contains("Gate (`verify.gate`): passed on the final commit"),
18559 "{body}"
18560 );
18561 assert!(
18563 state.reviews[0]
18564 .fix
18565 .as_ref()
18566 .unwrap()
18567 .notes
18568 .contains("could not commit")
18569 );
18570 }
18571
18572 #[test]
18573 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18574 let mut state = review_state(&["feat: x"]);
18575 let mut r1 = round_with_notes(1, Some(SANDBOXED));
18576 r1.fix.as_mut().unwrap().committed = false;
18577 let mut r2 = passing_round(2, Some(FINAL));
18578 r2.e2e[0].code = Some(1);
18579 state.reviews = vec![r1, r2];
18580 let body = body_at(&state);
18581 assert!(!body.contains("could not commit"), "{body}");
18582 assert!(!body.contains("could not run"), "{body}");
18583 assert!(body.contains("This round produced no commit"), "{body}");
18584 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18585 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18586 assert!(!body.contains("passed"), "{body}");
18587 }
18588
18589 #[test]
18590 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18591 let mut state = review_state(&["feat: x"]);
18592 state.reviews = vec![
18593 round_with_notes(1, Some(SANDBOXED)),
18594 passing_round(2, Some("0123456789abcdef")),
18595 ];
18596 ok_gate(&mut state, Some("0123456789abcdef"));
18597 let body = body_at(&state);
18598 assert!(!body.contains("HTTP tests could not run"), "{body}");
18599 assert!(!body.contains("could not commit"), "{body}");
18601 assert!(
18602 body.contains("passed on an earlier commit (`0123456`)"),
18603 "{body}"
18604 );
18605 assert!(!body.contains("passed on the final commit"), "{body}");
18606
18607 ok_gate(&mut state, None);
18609 let body = body_at(&state);
18610 assert!(
18611 body.contains("passed on a commit that was not recorded"),
18612 "{body}"
18613 );
18614 }
18615
18616 #[test]
18617 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18618 let state = state_with_summary(
18619 "add retries to the client",
18620 "TITLE: feat: retries\n\nDid it.",
18621 );
18622 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18623 assert_eq!(m.title, "feat: retries");
18624 assert!(
18625 m.body.contains("<summary>Original task</summary>"),
18626 "{}",
18627 m.body
18628 );
18629 assert!(!m.body.contains("Commits under review"), "{}", m.body);
18630 assert!(
18631 m.body
18632 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18633 "{}",
18634 m.body
18635 );
18636 }
18637
18638 #[test]
18639 fn pr_message_review_bounds_a_long_english_subject() {
18640 let long = format!("feat: {}", "word ".repeat(100));
18641 let m = pr_message(&review_state(&[&long]), 'A');
18642 assert!(m.title.starts_with("feat: word"), "{}", m.title);
18643 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18644 }
18645
18646 #[test]
18647 fn pr_message_implementation_run_is_unchanged_by_review_support() {
18648 let state = state_with_summary("add retries\n\ndetails", "- did some things");
18649 let m = pr_message(&state, 'A');
18650 assert_eq!(m.title, "add retries");
18651 assert!(m.body.contains("<summary>Original task</summary>"));
18652 assert!(!m.body.contains("Commits under review"));
18653 assert_eq!(landing_subject_source(&state), state.instruction);
18654 }
18655
18656 #[test]
18657 fn review_run_squash_subject_is_the_change_not_the_prompt() {
18658 let state = review_state(&["feat: the change", "fix: typo"]);
18659 let source = landing_subject_source(&state);
18660 assert_eq!(land::merge_subject("", &source), "feat: the change");
18661 assert_eq!(
18662 land::merge_subject("magi: candidate A (uncommitted work)", &source),
18663 "feat: the change"
18664 );
18665 assert_eq!(
18667 land::merge_subject("feat: renamed by hand", &source),
18668 "feat: renamed by hand"
18669 );
18670 let blank = review_state(&["日本語"]);
18671 assert!(
18672 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18673 );
18674 }
18675
18676 #[test]
18677 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18678 let state = review_state(&["feat: the change"]);
18679 let source = landing_subject_source(&state);
18680 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18681 assert_eq!(
18682 land::merge_subject(landing_title(&state, old), &source),
18683 "feat: the change"
18684 );
18685 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18686 let task = state_with_summary("add retries", "");
18687 assert_eq!(landing_title(&task, old), old);
18688 }
18689
18690 #[test]
18691 fn pr_message_describes_the_change_not_the_task() {
18692 let state = state_with_summary(
18693 "今回やってほしいこと: results projector を直す",
18694 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18695 );
18696 let m = pr_message(&state, 'A');
18697 assert_eq!(m.title, "fix(web): batch the runs list reads");
18698 assert!(
18699 m.body.starts_with("## Summary\n\n- reads run.json once"),
18700 "{}",
18701 m.body
18702 );
18703 assert!(!m.body.contains("TITLE:"), "{}", m.body);
18704 let task_at = m.body.find("今回やってほしいこと").unwrap();
18705 let details_at = m.body.find("<details>").unwrap();
18706 assert!(
18707 details_at < task_at,
18708 "the task lives inside <details>: {}",
18709 m.body
18710 );
18711 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18712 assert!(m.body.contains("magi:candidate-a"));
18713 }
18714
18715 #[test]
18716 fn pr_message_falls_back_to_the_task_without_a_title_line() {
18717 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18718 let m = pr_message(&state, 'A');
18719 assert_eq!(m.title, "add retries");
18720 assert!(
18721 m.body.contains("## Summary\n\n- did some things"),
18722 "{}",
18723 m.body
18724 );
18725
18726 let none = RunState::new(
18727 PathBuf::from("/repo"),
18728 "main".to_owned(),
18729 "abc1234".to_owned(),
18730 "add retries".to_owned(),
18731 Config::default(),
18732 );
18733 let m = pr_message(&none, 'A');
18734 assert_eq!(m.title, "add retries");
18735 assert!(!m.body.contains("## Summary"), "{}", m.body);
18736 }
18737
18738 #[test]
18739 fn pr_message_refuses_the_candidate_commit_subject() {
18740 for bad in [
18741 "TITLE: magi: candidate A (uncommitted work)",
18742 "TITLE: chore: stuff (uncommitted work)",
18743 "TITLE: ",
18744 ] {
18745 let state = state_with_summary("add retries", bad);
18746 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18747 }
18748 }
18749
18750 #[test]
18751 fn pr_message_bounds_a_very_long_task_and_title() {
18752 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18753 let state = state_with_summary(&long, "- nothing");
18754 let m = pr_message(&state, 'A');
18755 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18756 assert!(!m.title.contains('\n'));
18757
18758 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18759 let m = pr_message(&state, 'A');
18760 assert!(m.title.starts_with("feat: "));
18761 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18762 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18763 }
18764
18765 fn long_title_of(instruction: &str) -> String {
18766 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18767 }
18768
18769 #[test]
18770 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18771 let first = "Make the landing path keep a readable title for long tasks";
18772 let line = format!(
18773 "{first}. {}",
18774 "And then keep going with more words ".repeat(20)
18775 );
18776 let t = long_title_of(&line);
18777 assert_eq!(t, first);
18778 assert!(!t.starts_with("chore: land"));
18779 }
18780
18781 #[test]
18782 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18783 let line = "word ".repeat(200);
18784 let t = long_title_of(&line);
18785 assert!(t.ends_with("word..."), "{t}");
18786 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18787 }
18788
18789 #[test]
18790 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18791 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18792 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18793 }
18794 }
18795
18796 #[test]
18797 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18798 let mut state = state_with_summary("再試行を追加する", "- nothing");
18799 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18800 state.config.graph.github_language = "ja".into();
18801 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18802 }
18803
18804 #[test]
18805 fn pr_message_title_limit_is_exact() {
18806 let at = "a".repeat(PR_TITLE_MAX);
18807 assert_eq!(long_title_of(&at), at);
18808 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18809 assert!(over.ends_with("..."), "{over}");
18810 assert_eq!(over.chars().count(), PR_TITLE_MAX);
18811 }
18812
18813 #[test]
18814 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18815 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18816 let t = long_title_of(&line);
18817 assert!(t.ends_with("..."), "{t}");
18818 assert!(t.is_ascii(), "{t}");
18819 }
18820
18821 #[test]
18822 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18823 let line = format!(
18824 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18825 "plus more filler words ".repeat(20)
18826 );
18827 let t = long_title_of(&line);
18828 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18829 }
18830
18831 #[test]
18832 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18833 let line = format!("\n\n# {}", "title words ".repeat(40));
18834 let state = state_with_summary(&line, "- nothing");
18835 let m = pr_message(&state, 'A');
18836 assert!(m.title.starts_with("title words"), "{}", m.title);
18837 assert_eq!(
18838 land::merge_subject(&m.title, &landing_subject_source(&state)),
18839 m.title
18840 );
18841 assert_eq!(
18843 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18844 "feat: renamed by hand"
18845 );
18846 }
18847
18848 #[tokio::test]
18849 async fn github_text_rewrite_is_bounded_and_falls_back() {
18850 crate::run::pin_test_home();
18851 for (reply, accepted) in [
18852 (
18853 "TITLE: fix: retries\nAdd retries for failed requests.",
18854 true,
18855 ),
18856 (
18857 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18858 false,
18859 ),
18860 (
18861 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18862 true,
18863 ),
18864 (
18865 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18866 true,
18867 ),
18868 (
18869 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18870 true,
18871 ),
18872 (
18873 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18874 false,
18875 ),
18876 ("TITLE: fix: retries\n", false),
18877 ] {
18878 let dir = tempfile::tempdir().unwrap();
18879 let mut runner = runner_at(RunStatus::Gating);
18880 runner.state = state_with_summary(
18881 "add retries",
18882 "TITLE: fix: retries\n日本語の説明を書きます。",
18883 );
18884 runner.state.repo = dir.path().to_owned();
18885 runner.state.candidates[0].worktree = dir.path().to_owned();
18886 let mut author = spec("alpha");
18887 author.command = vec![
18888 "sh".into(),
18889 "-c".into(),
18890 "printf '%s' \"$REWRITE_REPLY\"".into(),
18891 ];
18892 author.env.insert("REWRITE_REPLY".into(), reply.into());
18893 runner.state.config.agents = vec![author];
18894 seed_public_summary(&mut runner);
18895 let winner = runner.state.candidates[0].clone();
18896 let first = runner
18897 .guarded_pr_message(&winner, None, true)
18898 .await
18899 .unwrap();
18900 let message = if accepted {
18901 first.expect("a passing rewrite never asks")
18902 } else {
18903 assert!(first.is_none());
18905 assert!(runner.state.parked);
18906 let store = ask::Questions::open();
18907 let asked: Vec<_> = store
18908 .list()
18909 .into_iter()
18910 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18911 .collect();
18912 assert_eq!(asked.len(), 1);
18913 assert!(!asked[0].detail.contains("secret"));
18914 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18915 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18916 assert_eq!(
18917 asked[0]
18918 .choices
18919 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18920 rewrite.is_some()
18921 );
18922 store
18925 .update(&asked[0].id, |q| {
18926 q.abandon("test");
18927 Ok(())
18928 })
18929 .unwrap();
18930 let again = runner
18931 .guarded_pr_message(&winner, None, true)
18932 .await
18933 .unwrap()
18934 .expect("an abandoned question falls back");
18935 let n = store
18936 .list()
18937 .into_iter()
18938 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18939 .count();
18940 assert_eq!(n, 1, "asked once per rejected text");
18941 again
18942 };
18943 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18944 assert!(!message.body.contains("secret"));
18945 assert!(!message.body.contains("prefix"));
18946 assert!(!message.title.contains("secret"));
18947 assert_eq!(
18948 message.body.contains("Add retries for failed requests."),
18949 accepted
18950 );
18951 assert_eq!(
18952 runner.state.seats["github-text-rewrite"].turns, 1,
18953 "only one rewrite invocation"
18954 );
18955 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18956 if accepted {
18957 assert!(runner.state.github_text.is_none());
18958 assert!(!runner.state.parked);
18959 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18960 assert!(message.body.contains("## Final verification"));
18961 assert!(message.body.contains("Evidence: final diff"));
18962 assert!(
18963 message
18964 .body
18965 .contains(&format!("magi:run/{}", runner.state.id))
18966 );
18967 }
18968 if !accepted {
18969 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18970 assert!(
18971 message
18972 .body
18973 .contains(&format!("magi:run/{}", runner.state.id))
18974 );
18975 }
18976 }
18977 }
18978
18979 fn scripted(id: &str, reply: &str) -> AgentSpec {
18980 let mut a = spec(id);
18981 a.command = vec![
18982 "sh".into(),
18983 "-c".into(),
18984 "printf '%s' \"$REWRITE_REPLY\"".into(),
18985 ];
18986 a.env.insert("REWRITE_REPLY".into(), reply.into());
18987 a
18988 }
18989
18990 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18991 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18992
18993 fn seed_public_summary(runner: &mut Runner) {
18995 let repo = &runner.state.repo;
18996 let g = |args: &[&str]| {
18997 let out = std::process::Command::new("git")
18998 .args(args)
18999 .current_dir(repo)
19000 .quiet()
19001 .output()
19002 .unwrap();
19003 assert!(out.status.success(), "git {args:?}");
19004 String::from_utf8_lossy(&out.stdout).trim().to_owned()
19005 };
19006 g(&["init", "-q", "-b", "main"]);
19007 g(&[
19008 "-c",
19009 "user.name=Test",
19010 "-c",
19011 "user.email=test@example.invalid",
19012 "commit",
19013 "-q",
19014 "--allow-empty",
19015 "-m",
19016 "feat: fixture",
19017 ]);
19018 let head = g(&["rev-parse", "HEAD"]);
19019 g(&["update-ref", "refs/remotes/origin/main", &head]);
19020 let evidence = crate::pr_outcome::summary_evidence(
19021 &runner.state,
19022 crate::pr_outcome::FinalHead::At(&head),
19023 );
19024 let generated = &runner.state.candidates[0].summary;
19025 runner.state.change_summary = Some(crate::run::ChangeSummary {
19026 base: head.clone(),
19027 head,
19028 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
19029 title: summary_title(generated).unwrap(),
19030 body: summary_without_title(generated),
19031 });
19032 }
19033
19034 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19035 let mut runner = runner_at(RunStatus::Gating);
19036 runner.state = state_with_summary("add retries", JA_SUMMARY);
19037 if review_only {
19038 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19039 }
19040 runner.state.repo = dir.to_owned();
19041 runner.state.candidates[0].worktree = dir.to_owned();
19042 runner.state.config.agents = agents;
19043 seed_public_summary(&mut runner);
19044 runner
19045 }
19046
19047 fn gate_questions(runner: &Runner) -> usize {
19048 ask::Questions::open()
19049 .list()
19050 .into_iter()
19051 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19052 .count()
19053 }
19054
19055 #[tokio::test]
19056 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19057 crate::run::pin_test_home();
19058 let dir = tempfile::tempdir().unwrap();
19059 let mut runner = rewrite_runner(
19060 dir.path(),
19061 true,
19062 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19063 );
19064 runner.state.config.roles.language_judge =
19065 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19066 let winner = runner.state.candidates[0].clone();
19067 let msg = runner
19068 .guarded_pr_message(&winner, None, true)
19069 .await
19070 .unwrap()
19071 .expect("an English rewrite posts without asking");
19072 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19073 assert!(msg.body.contains("<details>kept</details>"));
19074 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19075 assert_eq!(gate_questions(&runner), 0);
19076 assert!(runner.state.github_text.is_none());
19077 assert!(!runner.state.seats.contains_key("impl-A"));
19078 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19079 }
19080
19081 #[tokio::test]
19082 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19083 crate::run::pin_test_home();
19084 let dir = tempfile::tempdir().unwrap();
19085 let mut runner = rewrite_runner(
19086 dir.path(),
19087 false,
19088 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19089 );
19090 runner.state.config.roles.language_judge =
19091 Some(crate::config::AgentChoice::One("beta".into()));
19092 runner.state.seats.insert(
19093 "impl-A".into(),
19094 SeatState::new("impl-A", "alpha", runner.state.seed),
19095 );
19096 let winner = runner.state.candidates[0].clone();
19097 let msg = runner
19098 .guarded_pr_message(&winner, None, true)
19099 .await
19100 .unwrap()
19101 .expect("rewritten");
19102 assert!(msg.body.contains("<details>kept</details>"));
19103 assert_eq!(runner.state.seats["impl-A"].turns, 0);
19104 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19105 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19106 }
19107
19108 #[tokio::test]
19109 async fn rewrite_reuses_a_resumable_implementer_session() {
19110 crate::run::pin_test_home();
19111 let dir = tempfile::tempdir().unwrap();
19112 let mut runner = rewrite_runner(
19113 dir.path(),
19114 false,
19115 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
19116 );
19117 runner.state.config.roles.language_judge =
19118 Some(crate::config::AgentChoice::One("beta".into()));
19119 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19120 seat.turns = 1;
19121 runner.state.seats.insert("impl-A".into(), seat);
19122 let winner = runner.state.candidates[0].clone();
19123 runner
19124 .guarded_pr_message(&winner, None, true)
19125 .await
19126 .unwrap()
19127 .expect("rewritten");
19128 assert_eq!(runner.state.seats["impl-A"].turns, 2);
19129 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
19130 }
19131
19132 #[tokio::test]
19133 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
19134 crate::run::pin_test_home();
19137 let dir = tempfile::tempdir().unwrap();
19138 let mut exhausted = spec("alpha");
19139 exhausted.command = vec!["false".into()];
19140 let mut runner = rewrite_runner(
19141 dir.path(),
19142 false,
19143 vec![exhausted, scripted("beta", EN_REWRITE)],
19144 );
19145 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19147 "alpha".into(),
19148 "beta".into(),
19149 ]));
19150 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19151 seat.turns = 1;
19152 runner.state.seats.insert("impl-A".into(), seat);
19153 let winner = runner.state.candidates[0].clone();
19154 let msg = runner
19155 .guarded_pr_message(&winner, None, true)
19156 .await
19157 .unwrap()
19158 .expect("the role rewrites after the implementer fails");
19159 assert!(msg.body.contains("`src/net.rs`"));
19160 assert_eq!(
19161 runner.state.seats["impl-A"].turns, 2,
19162 "the implementer was tried once"
19163 );
19164 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19165 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19166 }
19167
19168 #[tokio::test]
19169 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
19170 crate::run::pin_test_home();
19171 let dir = tempfile::tempdir().unwrap();
19172 let mut broken = spec("bad");
19173 broken.command = vec!["false".into()];
19174 let mut runner = rewrite_runner(
19175 dir.path(),
19176 true,
19177 vec![
19178 broken,
19179 scripted("noformat", "Add retries."),
19180 scripted("good", EN_REWRITE),
19181 ],
19182 );
19183 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19184 "bad".into(),
19185 "noformat".into(),
19186 "good".into(),
19187 ]));
19188 let winner = runner.state.candidates[0].clone();
19189 let msg = runner
19190 .guarded_pr_message(&winner, None, true)
19191 .await
19192 .unwrap()
19193 .expect("the third agent rewrites");
19194 assert!(msg.body.contains("`src/net.rs`"));
19195 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
19196 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
19197 }
19198
19199 #[tokio::test]
19200 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
19201 crate::run::pin_test_home();
19202 for (agents, role, saved) in [
19204 (
19205 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
19206 Some("beta"),
19207 true,
19208 ),
19209 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
19210 (Vec::new(), None, false),
19211 ] {
19212 let dir = tempfile::tempdir().unwrap();
19213 let mut runner = rewrite_runner(dir.path(), true, agents);
19214 runner.state.config.roles.language_judge =
19215 role.map(|r| crate::config::AgentChoice::One(r.into()));
19216 let winner = runner.state.candidates[0].clone();
19217 let first = runner
19218 .guarded_pr_message(&winner, None, true)
19219 .await
19220 .unwrap();
19221 assert!(first.is_none(), "withheld text parks the run");
19222 assert!(runner.state.parked);
19223 assert_eq!(gate_questions(&runner), 1);
19224 let g = runner.state.github_text.as_ref().unwrap();
19225 assert_eq!(g.rewrite.is_some(), saved);
19226 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
19227 }
19228 }
19229
19230 #[tokio::test]
19231 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
19232 crate::run::pin_test_home();
19233 let dir = tempfile::tempdir().unwrap();
19234 let mut runner = runner_at(RunStatus::Gating);
19235 runner.state =
19236 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
19237 runner.state.repo = dir.path().to_owned();
19238 runner.state.candidates[0].worktree = dir.path().to_owned();
19239 let mut author = spec("alpha");
19240 author.command = vec![
19241 "sh".into(),
19242 "-c".into(),
19243 "printf '%s' \"$REWRITE_REPLY\"".into(),
19244 ];
19245 author.env.insert(
19246 "REWRITE_REPLY".into(),
19247 format!(
19248 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19249 "English sentence. ".repeat(600)
19250 ),
19251 );
19252 runner.state.config.agents = vec![author];
19253 seed_public_summary(&mut runner);
19254 let winner = runner.state.candidates[0].clone();
19255 assert!(
19256 runner
19257 .guarded_pr_message(&winner, None, true)
19258 .await
19259 .unwrap()
19260 .is_none()
19261 );
19262 let record = runner.state.github_text.clone().unwrap();
19263 let rewrite = record.rewrite.clone().unwrap();
19264 assert_eq!(rewrite.failed, ["body-language"]);
19265 assert!(
19266 rewrite
19267 .body
19268 .ends_with(&format!("magi:run/{}", runner.state.id))
19269 );
19270 let store = ask::Questions::open();
19271 let q = store.get(record.question.as_ref().unwrap()).unwrap();
19272 assert!(
19273 q.choices
19274 .contains(&crate::github_text::USE_REWRITE.to_owned())
19275 );
19276 for label in [
19277 "ORIGINAL",
19278 "FALLBACK",
19279 "REWRITE",
19280 "Rules still failed: body-language",
19281 "truncated",
19282 ] {
19283 assert!(q.detail.contains(label), "{label}");
19284 }
19285 let artifact = std::fs::read_to_string(crate::run::artifact_path(
19286 &runner.state,
19287 &crate::github_text::artifact_name(&record.fingerprint),
19288 ))
19289 .unwrap();
19290 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19291 runner.state =
19293 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19294 store
19295 .update(&q.id, |q| {
19296 q.say("まだ日本語")?;
19297 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19298 })
19299 .unwrap();
19300 assert!(
19301 runner
19302 .guarded_pr_message(&winner, None, true)
19303 .await
19304 .unwrap()
19305 .is_none()
19306 );
19307 let retry = runner.state.github_text.as_ref().unwrap();
19308 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19309 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19310 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19311 assert!(
19312 q.posting_comparison
19313 .as_ref()
19314 .unwrap()
19315 .matches_detail(&q.detail)
19316 );
19317 store
19318 .update(&q.id, |q| {
19319 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19320 })
19321 .unwrap();
19322 let message = runner
19323 .guarded_pr_message(&winner, None, true)
19324 .await
19325 .unwrap()
19326 .unwrap();
19327 assert_eq!(message.title, rewrite.title);
19328 assert_eq!(message.body, rewrite.body);
19329 assert_eq!(
19330 runner.state.github_text.as_ref().unwrap().choice,
19331 Some(GithubTextChoice::Rewrite)
19332 );
19333 let again = runner
19334 .guarded_pr_message(&winner, None, true)
19335 .await
19336 .unwrap()
19337 .unwrap();
19338 assert_eq!(again.title, message.title);
19339 assert_eq!(again.body, message.body);
19340 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19341 assert!(
19342 !runner
19343 .state
19344 .events
19345 .iter()
19346 .any(|e| e.message.contains(&rewrite.title)
19347 || e.message.contains("English sentence."))
19348 );
19349 }
19350
19351 #[tokio::test]
19352 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19353 crate::run::pin_test_home();
19354 let dir = tempfile::tempdir().unwrap();
19355 let mut runner = runner_at(RunStatus::Gating);
19356 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19357 runner.state.repo = dir.path().to_owned();
19358 runner.state.candidates[0].worktree = dir.path().to_owned();
19359 seed_public_summary(&mut runner);
19360 let winner = runner.state.candidates[0].clone();
19361 assert!(
19362 runner
19363 .guarded_pr_message(&winner, None, true)
19364 .await
19365 .unwrap()
19366 .is_none()
19367 );
19368 let record = runner.state.github_text.as_ref().unwrap();
19369 assert!(record.rewrite.is_none());
19370 let mut old = serde_json::to_value(record).unwrap();
19372 old.as_object_mut().unwrap().remove("rewrite");
19373 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19374 assert!(old.rewrite.is_none());
19375 let id = record.question.clone().unwrap();
19376 ask::Questions::open()
19377 .update(&id, |q| {
19378 q.choices.push(crate::github_text::USE_REWRITE.into());
19379 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19380 })
19381 .unwrap();
19382 let message = runner
19383 .guarded_pr_message(&winner, None, true)
19384 .await
19385 .unwrap()
19386 .unwrap();
19387 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19388 assert_eq!(
19389 runner.state.github_text.as_ref().unwrap().choice,
19390 Some(GithubTextChoice::Fallback)
19391 );
19392 }
19393
19394 #[tokio::test]
19395 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19396 crate::run::pin_test_home();
19397 let dir = tempfile::tempdir().unwrap();
19398 let mut runner = runner_at(RunStatus::Gating);
19399 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19400 runner.state.repo = dir.path().to_owned();
19401 runner.state.candidates[0].worktree = dir.path().to_owned();
19402 seed_public_summary(&mut runner);
19403 let winner = runner.state.candidates[0].clone();
19404 assert!(
19405 runner
19406 .guarded_pr_message(&winner, None, true)
19407 .await
19408 .unwrap()
19409 .is_none()
19410 );
19411 let store = ask::Questions::open();
19412 let find = |run: &str| {
19413 store
19414 .list()
19415 .into_iter()
19416 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19417 .unwrap()
19418 };
19419 let run = runner.state.id.clone();
19420 let q = find(&run);
19421 let fp = runner
19422 .state
19423 .github_text
19424 .as_ref()
19425 .unwrap()
19426 .fingerprint
19427 .clone();
19428 let name = crate::github_text::artifact_name(&fp);
19429 let path = crate::run::artifact_path(&runner.state, &name);
19430 assert!(q.detail.contains(&path.display().to_string()));
19431 assert!(q.detail.contains("title-language: title"));
19432 let saved = std::fs::read_to_string(&path).expect("artifact written");
19433 assert!(saved.contains("日本語のタイトル"));
19434 assert!(q.cwd.is_none());
19435 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19436 let say = |id: &str, text: &str, choice: &str| {
19438 store
19439 .update(id, |q| {
19440 q.thread.push(ask::Turn {
19441 who: ask::Who::Operator,
19442 body: text.to_owned(),
19443 at: jiff::Timestamp::now(),
19444 note: None,
19445 });
19446 q.answer(ask::Answer::Choice(choice.to_owned()))
19447 })
19448 .unwrap();
19449 };
19450 say(&q.id, "まだ日本語", "use my text");
19451 assert!(
19452 runner
19453 .guarded_pr_message(&winner, None, true)
19454 .await
19455 .unwrap()
19456 .is_none()
19457 );
19458 let second = find(&run);
19459 assert_ne!(second.id, q.id);
19460 assert!(second.detail.contains("did not pass"));
19461 assert!(second.detail.contains("日本語のタイトル"));
19464 assert!(!second.detail.contains("まだ日本語"));
19465 assert!(second.detail.contains(&path.display().to_string()));
19466 say(&second.id, "fix: retry failed requests", "use my text");
19467 let message = runner
19468 .guarded_pr_message(&winner, None, true)
19469 .await
19470 .unwrap()
19471 .expect("a vetted replacement is posted");
19472 assert_eq!(message.title, "fix: retry failed requests");
19473 assert_eq!(
19474 runner
19475 .state
19476 .github_text
19477 .as_ref()
19478 .unwrap()
19479 .chosen_title
19480 .as_deref(),
19481 Some("fix: retry failed requests")
19482 );
19483 assert!(runner.state.github_text.as_ref().unwrap().resolved);
19484 let again = runner
19486 .guarded_pr_message(&winner, None, true)
19487 .await
19488 .unwrap()
19489 .unwrap();
19490 assert_eq!(again.title, message.title);
19491 assert_eq!(
19492 store
19493 .list()
19494 .into_iter()
19495 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19496 .count(),
19497 2
19498 );
19499 }
19500
19501 #[test]
19502 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19503 let mut state = state_with_summary(
19507 "add retries",
19508 "TITLE: fix(web): batch reads\n- reads run.json once",
19509 );
19510 state.config.graph.language = "ja".to_owned();
19511 let m = pr_message(&state, 'A');
19512 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19513 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19514
19515 let task = "今回やってほしいこと: results projector を直す";
19518 let mut state = state_with_summary(task, "- no title line");
19519 state.config.graph.language = "ja".to_owned();
19520 let m = pr_message(&state, 'A');
19521 assert_eq!(
19522 m.title,
19523 format!("chore: land candidate A of run {}", state.id)
19524 );
19525 assert!(
19526 m.body.contains(&format!(
19527 "<summary>Original task</summary>\n\n{task}\n\n</details>"
19528 )),
19529 "{}",
19530 m.body
19531 );
19532 }
19533
19534 #[test]
19535 fn pr_message_scrubs_home_paths_and_addresses() {
19536 let state = state_with_summary(
19537 "fix it in /Users/someone/src/x",
19538 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19539 );
19540 let m = pr_message(&state, 'A');
19541 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19542 assert!(!m.body.contains(leak), "{}", m.body);
19543 }
19544 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19545 }
19546
19547 #[test]
19548 fn pr_message_survives_a_task_that_closes_details() {
19549 let state = state_with_summary("a </details> b", "TITLE: fix: x");
19550 let m = pr_message(&state, 'A');
19551 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19552 }
19553
19554 #[test]
19555 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19556 let cmd = manual_merge_command(
19557 MergeStyle::Squash,
19558 Path::new("/repo"),
19559 "b",
19560 "fix: \"quoted\" $(x) `y`\n\nbody",
19561 );
19562 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19563 }
19564
19565 #[test]
19566 fn manual_merge_command_matches_the_configured_style() {
19567 let repo = Path::new("/repo");
19568 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19569
19570 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19571 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19572
19573 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19574 assert_eq!(
19575 squash,
19576 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19577 \"Merge magi run 0832 (candidate A)\""
19578 );
19579
19580 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19581 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19582 }
19583
19584 #[test]
19585 fn a_nudge_gets_a_quarter_of_the_budget() {
19586 assert_eq!(retry_budget(secs(1200), true), secs(300));
19588 assert_eq!(retry_budget(secs(3600), true), secs(900));
19589 }
19590
19591 #[test]
19592 fn a_resent_prompt_keeps_the_whole_budget() {
19593 assert_eq!(retry_budget(secs(1200), false), secs(1200));
19596 assert_eq!(retry_budget(secs(60), false), secs(60));
19597 }
19598
19599 #[test]
19600 fn the_floor_never_exceeds_the_original_budget() {
19601 assert_eq!(retry_budget(secs(60), true), secs(60));
19605 assert_eq!(retry_budget(secs(480), true), secs(120));
19606 assert_eq!(retry_budget(secs(0), true), secs(0));
19607 }
19608
19609 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19610 agent::CommandEvidence {
19611 id: "item1".to_owned(),
19612 description: "cargo test".to_owned(),
19613 exit_code,
19614 result_summary: String::new(),
19615 source: "codex".to_owned(),
19616 }
19617 }
19618
19619 #[test]
19620 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19621 assert!(!has_unconfirmed_command(&[]));
19625 }
19626
19627 #[test]
19628 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19629 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19633 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19634 assert!(!has_unconfirmed_command(&[
19635 evidence(Some(0)),
19636 evidence(Some(101))
19637 ]));
19638 }
19639
19640 #[test]
19641 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19642 assert!(has_unconfirmed_command(&[
19643 evidence(Some(0)),
19644 evidence(None)
19645 ]));
19646 }
19647
19648 #[test]
19649 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19650 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19651 assert_eq!(
19652 verified_noop_claim(true, &[], text).as_deref(),
19653 Some("already fixed by b32cfc4, on main.")
19654 );
19655 }
19656
19657 #[test]
19658 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19659 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19662 assert!(verified_noop_claim(false, &[], text).is_none());
19663 }
19664
19665 #[test]
19666 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19667 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19668 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19669 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19671 }
19672
19673 #[test]
19674 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19675 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19676 }
19677
19678 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19681 runner.state.candidates = shape
19682 .iter()
19683 .enumerate()
19684 .map(|(i, &(empty, verified))| Candidate {
19685 index: i,
19686 label: (b'A' + i as u8) as char,
19687 agent: "sonnet".to_owned(),
19688 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19689 worktree: PathBuf::from(format!("/wt/{i}")),
19690 summary: String::new(),
19691 stat: String::new(),
19692 files: 0,
19693 commits: 0,
19694 empty,
19695 failed: None,
19696 verified_noop: verified.map(str::to_owned),
19697 duration_ms: 0,
19698 folded: false,
19699 })
19700 .collect();
19701 }
19702
19703 #[test]
19704 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19705 ask_test_home();
19706 let mut runner = runner_at(RunStatus::Implementing);
19707 set_candidates(
19708 &mut runner,
19709 &[
19710 (true, Some("already on main at b32cfc4")),
19711 (true, Some("same fix, see the existing test")),
19712 ],
19713 );
19714
19715 runner
19716 .after_implement()
19717 .expect("a verified no-op is not an error");
19718
19719 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19720 }
19721
19722 #[test]
19723 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19724 ask_test_home();
19725 let mut runner = runner_at(RunStatus::Implementing);
19726 set_candidates(
19730 &mut runner,
19731 &[(true, Some("already on main at b32cfc4")), (true, None)],
19732 );
19733
19734 let err = runner
19735 .after_implement()
19736 .expect_err("an unverified empty candidate must still fail the run");
19737
19738 assert!(
19739 err.to_string().contains("no candidate produced a change"),
19740 "{err}"
19741 );
19742 assert_eq!(runner.state.status, RunStatus::Failed);
19743 }
19744
19745 #[test]
19746 fn after_implement_still_fails_an_ordinary_all_empty_run() {
19747 ask_test_home();
19748 let mut runner = runner_at(RunStatus::Implementing);
19749 set_candidates(&mut runner, &[(true, None), (true, None)]);
19750
19751 let err = runner
19752 .after_implement()
19753 .expect_err("no candidate declared anything; this is an ordinary failure");
19754
19755 assert!(
19756 err.to_string().contains("no candidate produced a change"),
19757 "{err}"
19758 );
19759 assert_eq!(runner.state.status, RunStatus::Failed);
19760 }
19761}
19762
19763#[derive(Debug)]
19766struct GraftRestoreFailed(String);
19767
19768impl std::fmt::Display for GraftRestoreFailed {
19769 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19770 write!(
19771 f,
19772 "could not restore the winner to its pre-graft commit: {}",
19773 self.0
19774 )
19775 }
19776}
19777
19778impl std::error::Error for GraftRestoreFailed {}
19779
19780#[cfg(test)]
19784mod resource_wait_tests {
19785 use super::*;
19786 use std::collections::BTreeMap;
19787
19788 fn sh(id: &str, script: &str) -> AgentSpec {
19789 AgentSpec {
19790 id: id.to_owned(),
19791 kind: crate::config::AgentKind::Command,
19792 model: None,
19793 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19794 extra_args: Vec::new(),
19795 env: BTreeMap::new(),
19796 prompt_delivery: None,
19797 max_turn_secs: None,
19798 }
19799 }
19800
19801 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19802 SeatJob {
19803 seat: SeatState::new("impl-A", &spec.id, 1),
19804 spec,
19805 cwd: dir.to_path_buf(),
19806 prompt: "do it".to_owned(),
19807 timeout,
19808 allow_write: true,
19809 sessions: false,
19810 artifacts: dir.join("art"),
19811 stem: "impl-A".to_owned(),
19812 handover: None,
19813 }
19814 }
19815
19816 fn state(resource_wait_secs: u64) -> RunState {
19817 let mut config = Config::default();
19818 config.graph.timeout_resource_wait = resource_wait_secs;
19819 RunState::new(
19820 PathBuf::from("/repo"),
19821 "main".to_owned(),
19822 "abc1234".to_owned(),
19823 "task".to_owned(),
19824 config,
19825 )
19826 }
19827
19828 fn hold(cache: &Path) -> crate::cache::Guard {
19829 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19830 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19831 crate::cache::AcquireOutcome::Acquired(g) => g,
19832 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19833 }
19834 }
19835
19836 async fn run(
19837 jobs: Vec<SeatJob>,
19838 sem: Arc<Semaphore>,
19839 cache: Option<&Path>,
19840 st: &mut RunState,
19841 ) -> Vec<(usize, SeatState, AgentOutcome)> {
19842 let prompts = Prompts::default();
19843 let ctx = WaveCtx {
19844 run: "r",
19845 node: "implement",
19846 prompts: &prompts,
19847 cache,
19848 round: None,
19849 carry_seats: false,
19850 pauses: None,
19851 };
19852 wave(jobs, sem, &ctx, st, 0).await
19853 }
19854
19855 #[tokio::test]
19856 async fn verification_cache_wait_preserves_the_command_budget() {
19857 crate::run::pin_test_home();
19858 let tmp = tempfile::tempdir().unwrap();
19859 let cache = tmp.path().join("target");
19860 let guard = hold(&cache);
19861 let release = tokio::spawn(async move {
19862 tokio::time::sleep(Duration::from_millis(100)).await;
19863 guard.release();
19864 });
19865 let mut st = state(10);
19866 let budget = Duration::from_millis(50);
19867 let wait = repository_wait(&st, None);
19868 let (outcomes, _) = with_cache_lease(
19869 &mut st,
19870 Some(&cache),
19871 "gate",
19872 "gate",
19873 tmp.path(),
19874 "abc1234",
19875 budget,
19876 "test gate",
19877 &wait,
19878 |_, available| async move {
19879 assert_eq!(
19880 available, budget,
19881 "contention must not shorten verification"
19882 );
19883 (Vec::new(), false, Vec::new())
19884 },
19885 )
19886 .await
19887 .unwrap();
19888 release.await.unwrap();
19889 assert!(outcomes.is_empty(), "{outcomes:?}");
19890 assert!(
19891 st.events
19892 .iter()
19893 .any(|event| event.message.contains("waiting for"))
19894 );
19895 assert!(st.resource_wait.is_none());
19896 hold(&cache).release();
19898 }
19899
19900 #[tokio::test]
19901 async fn verification_cache_wait_expires_without_starting_commands() {
19902 crate::run::pin_test_home();
19903 let tmp = tempfile::tempdir().unwrap();
19904 let cache = tmp.path().join("target");
19905 let _held = hold(&cache);
19906 let mut st = state(0);
19907 let wait = repository_wait(&st, None);
19908 let (outcomes, _) = with_cache_lease(
19909 &mut st,
19910 Some(&cache),
19911 "gate",
19912 "gate",
19913 tmp.path(),
19914 "abc1234",
19915 Duration::from_secs(60),
19916 "test gate",
19917 &wait,
19918 |_, _| async { panic!("commands must not start without the cache lease") },
19919 )
19920 .await
19921 .unwrap();
19922 assert_eq!(outcomes.len(), 1);
19923 assert!(outcomes[0].resource_blocked);
19924 assert!(st.resource_wait.is_none());
19925 assert!(matches!(
19926 st.resource_block.unwrap().cause,
19927 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19928 ));
19929 }
19930
19931 #[tokio::test]
19932 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19933 crate::run::pin_test_home();
19934 let tmp = tempfile::tempdir().unwrap();
19935 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19936 let cache = tmp.path().join("cache");
19937 let mut state = state(21600);
19938 let wait = repository_wait(&state, None);
19939 let (outcomes, _) = with_cache_lease(
19940 &mut state,
19941 Some(&cache),
19942 "gate",
19943 "gate",
19944 tmp.path(),
19945 "head",
19946 Duration::from_secs(60),
19947 "gate",
19948 &wait,
19949 |_, _| async { panic!("freshness failure must not start verification") },
19950 )
19951 .await
19952 .unwrap();
19953 assert_eq!(outcomes.len(), 1);
19954 assert!(outcomes[0].resource_blocked);
19955 let block = state.resource_block.unwrap();
19956 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19957 assert!(!block.description().contains("timed out"));
19958 }
19959
19960 #[tokio::test]
19961 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19962 crate::run::pin_test_home();
19963 let tmp = tempfile::tempdir().unwrap();
19964 let cache = tmp.path().join("cache");
19965 let _held = hold(&cache);
19966 let mut state = state(21600);
19967 let pause = Pause::new();
19968 let interrupt = Pause::new();
19969 let cause = ParkCause::Operator;
19970 pause.park_for(cause.clone());
19971 let prompts = Prompts::default();
19972 let ctx = WaveCtx {
19973 run: "test",
19974 node: "fix",
19975 prompts: &prompts,
19976 cache: Some(&cache),
19977 round: None,
19978 carry_seats: false,
19979 pauses: Some((&pause, &interrupt)),
19980 };
19981 let result = wave(
19982 vec![job(
19983 sh("writer", "exit 99"),
19984 tmp.path(),
19985 Duration::from_secs(1),
19986 )],
19987 Arc::new(Semaphore::new(1)),
19988 &ctx,
19989 &mut state,
19990 0,
19991 )
19992 .await;
19993 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19994 assert_eq!(result[0].1.turns, 0);
19995 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19996 assert_eq!(
19997 state.resource_block.unwrap().cause,
19998 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19999 );
20000 assert!(state.active.is_empty());
20001 }
20002
20003 #[tokio::test]
20004 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
20005 crate::run::pin_test_home();
20006 let tmp = tempfile::tempdir().unwrap();
20007 let cache = tmp.path().join("target-a");
20008 let guard = hold(&cache);
20009 tokio::spawn(async move {
20011 tokio::time::sleep(Duration::from_millis(1500)).await;
20012 guard.release();
20013 });
20014 let mut st = state(60);
20015 let results = run(
20016 vec![job(
20017 sh("alpha", "echo ok"),
20018 tmp.path(),
20019 Duration::from_secs(1),
20020 )],
20021 Arc::new(Semaphore::new(1)),
20022 Some(&cache),
20023 &mut st,
20024 )
20025 .await;
20026 assert!(
20027 matches!(results[0].2, AgentOutcome::Ok(_)),
20028 "the wait must not be charged to the agent"
20029 );
20030 assert!(st.resource_block.is_none());
20031 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20032 assert!(st.resource_wait.is_none() && st.active.is_empty());
20033 }
20034
20035 #[tokio::test]
20036 #[cfg(unix)]
20037 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20038 use std::os::fd::AsRawFd as _;
20039 crate::run::pin_test_home();
20040 let tmp = tempfile::tempdir().unwrap();
20041 crate::git::git(tmp.path(), &["init"]).await.unwrap();
20042 let common = tmp.path().join(".git");
20043 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20044 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20045 assert_eq!(
20046 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20047 0
20048 );
20049 let mut st = state(1);
20050 st.config.coordination = crate::config::test_coordination();
20051 let mut reader = job(
20052 sh("reader", "echo must-not-run"),
20053 tmp.path(),
20054 Duration::from_secs(30),
20055 );
20056 reader.allow_write = false;
20057 let results = run(
20058 vec![reader.clone()],
20059 Arc::new(Semaphore::new(1)),
20060 None,
20061 &mut st,
20062 )
20063 .await;
20064 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20065 assert_eq!(results[0].1.turns, 0);
20066 assert!(FailClass::of(&results[0].2).is_none());
20067 assert_eq!(
20068 st.resource_block.as_ref().unwrap().kind,
20069 "repository coordination"
20070 );
20071 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20072 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20073 drop(external);
20074 let editor = crate::repo_lock::SeatLease::for_checkout(
20075 tmp.path(),
20076 true,
20077 crate::config::test_coordination().as_ref(),
20078 )
20079 .await
20080 .unwrap();
20081 let pause = Pause::new();
20082 let interrupt = Pause::new();
20083 let prompts = Prompts::default();
20084 let ctx = WaveCtx {
20085 run: "r",
20086 node: "review",
20087 prompts: &prompts,
20088 cache: None,
20089 round: None,
20090 carry_seats: false,
20091 pauses: Some((&pause, &interrupt)),
20092 };
20093 let (results, ()) = tokio::join!(
20094 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20095 async {
20096 tokio::time::sleep(Duration::from_millis(50)).await;
20097 pause.park();
20098 }
20099 );
20100 assert!(
20101 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20102 );
20103 assert_eq!(results[0].1.turns, 0);
20104 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20105 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20106 drop(editor);
20107 }
20108
20109 #[tokio::test]
20110 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20111 crate::run::pin_test_home();
20112 let tmp = tempfile::tempdir().unwrap();
20113 let cache = tmp.path().join("target-b");
20114 let _held = hold(&cache);
20115 let mut st = state(1);
20116 let results = run(
20117 vec![job(
20118 sh("alpha", "echo ok"),
20119 tmp.path(),
20120 Duration::from_secs(1),
20121 )],
20122 Arc::new(Semaphore::new(1)),
20123 Some(&cache),
20124 &mut st,
20125 )
20126 .await;
20127 let (_, seat, out) = &results[0];
20128 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
20129 assert!(wave_blocked(&results));
20130 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
20131 assert_eq!(seat.turns, 0, "nothing was asked");
20132 assert!(
20133 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
20134 "no agent was spawned"
20135 );
20136 assert!(FailClass::of(out).is_none(), "not a failure class");
20137 let block = st.resource_block.as_ref().expect("recorded for the resume");
20138 assert_eq!(block.kind, "shared build cache");
20139 assert!(matches!(
20140 block.cause,
20141 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
20142 ));
20143 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20144 assert!(st.active.is_empty());
20145 }
20146
20147 #[tokio::test]
20148 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
20149 crate::run::pin_test_home();
20150 let tmp = tempfile::tempdir().unwrap();
20151 let mut st = state(1);
20152 let results = run(
20153 vec![job(
20154 sh("alpha", "echo ok"),
20155 tmp.path(),
20156 Duration::from_secs(1),
20157 )],
20158 Arc::new(Semaphore::new(0)),
20159 None,
20160 &mut st,
20161 )
20162 .await;
20163 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20164 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
20165 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
20166 }
20167
20168 #[tokio::test]
20169 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
20170 crate::run::pin_test_home();
20171 let tmp = tempfile::tempdir().unwrap();
20172 let mut st = state(60);
20173 let results = run(
20174 vec![job(
20175 sh("alpha", "sleep 30"),
20176 tmp.path(),
20177 Duration::from_secs(1),
20178 )],
20179 Arc::new(Semaphore::new(1)),
20180 None,
20181 &mut st,
20182 )
20183 .await;
20184 let out = &results[0].2;
20185 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
20186 let class = FailClass::of(out).expect("a failure");
20187 assert_eq!(class, FailClass::Timeout);
20188 assert!(should_hand_over(None, &class), "existing handover policy");
20189 assert!(st.resource_block.is_none());
20190 }
20191}