1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530enum FixStep {
532 Resource,
533 Recorded { report_lost: bool },
534}
535
536struct FixClaim {
553 path: PathBuf,
554}
555
556impl FixClaim {
557 fn acquire(dir: &Path) -> Result<Self> {
558 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559 let path = dir.join("fix.lock");
560 let retry = crate::lock_file::Retry::new(None);
561 match Self::create(&path, retry) {
562 Ok(claim) => Ok(claim),
563 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564 if Self::reclaim_if_dead(&path) {
565 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566 } else {
567 bail!(
568 "another repair or `magi fix` is already running for this run ({} exists)",
569 path.display()
570 )
571 }
572 }
573 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574 }
575 }
576
577 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578 let mut f = retry.create(path)?;
579 use std::io::Write as _;
580 writeln!(f, "{}", std::process::id())?;
582 Ok(Self {
583 path: path.to_owned(),
584 })
585 }
586
587 fn reclaim_if_dead(path: &Path) -> bool {
591 let dead = std::fs::read_to_string(path)
592 .ok()
593 .and_then(|body| body.trim().parse::<u32>().ok())
594 .is_some_and(|pid| !crate::proc::pid_alive(pid));
595 dead && std::fs::remove_file(path).is_ok()
596 }
597}
598
599impl Drop for FixClaim {
600 fn drop(&mut self) {
601 let _ = std::fs::remove_file(&self.path);
602 }
603}
604
605struct DriverExecution<'a> {
608 runner: &'a mut Runner,
609 driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613 fn drop(&mut self) {
614 self.driver.release_local(&mut self.runner.state);
615 }
616}
617
618impl Runner {
619 pub async fn start(
621 repo: &Path,
622 instruction: String,
623 config: Config,
624 origin: Origin,
625 ) -> Result<Self> {
626 Self::start_naming(repo, instruction, "", config, origin).await
627 }
628
629 pub async fn start_naming(
632 repo: &Path,
633 instruction: String,
634 _also_scan: &str,
635 config: Config,
636 origin: Origin,
637 ) -> Result<Self> {
638 let repo = git::toplevel(repo).await?;
639 let missing = agent::missing_programs(&config.agents);
640 if !missing.is_empty() {
641 bail!(
642 "these agent programs are not on PATH: {}. Fix the roster in \
643 magi.toml or install them.",
644 missing.join(", ")
645 );
646 }
647 let base_branch =
648 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649 .await?;
650 let fetched_base = resolve_base(
651 &repo,
652 &base_branch,
653 &config.merge.remote,
654 config.coordination.as_ref(),
655 )
656 .await?;
657 let base_commit = fetched_base.commit.clone();
658 let roles = config.resolve_roles()?;
659 let max_parallel = config.graph.max_parallel.max(1);
660 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661 state.fetched_base = Some(fetched_base);
665 state.origin = Some(origin);
666 state.reserve()?;
667 state.event("start", format!("run {} created", state.id));
668 state.save()?;
669 Ok(Self {
670 state,
671 roles,
672 sem: Arc::new(Semaphore::new(max_parallel)),
673 pause: Pause::new(),
674 interrupt: Pause::new(),
675 })
676 }
677
678 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692 Self::review_taking_over(repo, branch, config, None, origin).await
693 }
694
695 pub async fn review_taking_over(
702 repo: &Path,
703 branch: &str,
704 config: Config,
705 takeover: Option<crate::handover::Takeover>,
706 origin: Origin,
707 ) -> Result<Self> {
708 let repo = git::toplevel(repo).await?;
709 let missing = agent::missing_programs(&config.agents);
710 if !missing.is_empty() {
711 bail!(
712 "these agent programs are not on PATH: {}. Fix the roster in \
713 magi.toml or install them.",
714 missing.join(", ")
715 );
716 }
717 let base_branch =
718 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719 .await?;
720 if base_branch == branch {
721 bail!("`{branch}` is the base branch; there is nothing to review against");
722 }
723 let fetched_base = resolve_base(
724 &repo,
725 &base_branch,
726 &config.merge.remote,
727 config.coordination.as_ref(),
728 )
729 .await?;
730 let base_commit = fetched_base.commit.clone();
731
732 let roles = config.resolve_roles()?;
733 let max_parallel = config.graph.max_parallel.max(1);
734 let mut state = RunState::new(
735 repo.clone(),
736 base_branch,
737 base_commit.clone(),
738 String::new(),
739 config,
740 );
741 state.fetched_base = Some(fetched_base);
742 state.origin = Some(origin);
743 state.reserve()?;
744
745 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752 earlier: Vec::new(),
753 home: crate::run::home(),
754 choice: None,
755 task_instruction: None,
756 });
757 let wait = repository_wait(&state, None);
758 let coordination_config = state.config.coordination.clone();
759 let coordination =
760 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761 .await?;
762 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763 if let Some(released) = &released {
764 state.event(
765 "release",
766 format!(
767 "took `{branch}` over from run {}: its worktree was released: {}",
768 crate::run::short_of(&released.old_id),
769 released.audit
770 ),
771 );
772 }
773 if let Some(choice) = takeover.choice.as_ref()
777 && let Err(e) =
778 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779 .await
780 {
781 if let Some(released) = &released {
782 released.restore(&repo, branch).await;
783 }
784 return Err(e.context("applying the owner's answer about the diverged branch"));
785 }
786 for id in takeover.earlier.iter().rev() {
792 if let Ok(old) = RunState::load_under(id, &takeover.home)
793 && let Some(head) = old.pr_head.clone()
794 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795 {
796 state.pr_head = Some(head);
797 break;
798 }
799 }
800 drop(coordination);
801 let opened = Self::open_review(
802 &repo,
803 branch,
804 state,
805 roles,
806 max_parallel,
807 base_commit,
808 &takeover,
809 )
810 .await;
811 if opened.is_err()
812 && let Some(released) = &released
813 {
814 let _coordination = crate::repo_lock::SeatLease::operation_with(
815 &repo,
816 coordination_config.as_ref(),
817 &wait,
818 )
819 .await?;
820 released.restore(&repo, branch).await;
821 }
822 opened
823 }
824
825 pub async fn recover_pr(
827 repo: &Path,
828 task: &str,
829 selection: crate::pr_recovery::Selection,
830 mut config: Config,
831 ) -> Result<Self> {
832 selection.configure(&mut config)?;
833 selection.check_task(task, &selection.instruction)?;
834 let base_branch =
835 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836 .await?;
837 let fetched_base = resolve_base(
838 repo,
839 &base_branch,
840 &config.merge.remote,
841 config.coordination.as_ref(),
842 )
843 .await?;
844 let base_commit = fetched_base.commit.clone();
845 selection
846 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847 .await?;
848 Box::pin(selection.verify_source(repo)).await?;
849 if let Some(proof) = &selection.source_proof {
850 anyhow::ensure!(
851 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852 "source rebase base is outside the fetched landing base"
853 );
854 }
855 let roles = config.resolve_roles()?;
856 let max_parallel = config.graph.max_parallel.max(1);
857 let mut state = RunState::new(
858 repo.to_path_buf(),
859 base_branch,
860 base_commit.clone(),
861 selection.instruction.clone(),
862 config,
863 );
864 state.fetched_base = Some(fetched_base);
865 state.origin = Some(crate::run::Origin::queue(task));
866 state.pr_head = Some(selection.branch.clone());
867 state.pr_recovery = Some(selection.clone());
868 state.reserve()?;
869 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870 state.discard_reserved();
872 return Err(e);
873 }
874 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875 }
876
877 async fn open_selected(
878 repo: &Path,
879 state: RunState,
880 roles: ResolvedRoles,
881 max_parallel: usize,
882 base_commit: String,
883 selection: crate::pr_recovery::Selection,
884 ) -> Result<Self> {
885 let mut state = state;
886 let branch = state.branch_for('A');
887 let wait = repository_wait(&state, None);
888 let prepared = async {
889 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890 git::git(repo, &["branch", &branch, selection.source()]).await?;
891 drop(coordination);
892 anyhow::Ok(())
893 }
894 .await;
895 if let Err(e) = prepared {
896 state.discard_reserved();
899 return Err(e);
900 }
901 state.event(
902 "recovery",
903 format!(
904 "selected implementation {} for {} (public lease {})",
905 selection.source(),
906 selection.url,
907 selection.head
908 ),
909 );
910 let takeover = crate::handover::Takeover {
911 earlier: Vec::new(),
912 home: crate::run::home(),
913 choice: None,
914 task_instruction: Some(selection.instruction.clone()),
915 };
916 if let Err(e) = state.save() {
917 state.discard_reserved();
918 return Err(e);
919 }
920 let id = state.id.clone();
921 let opened = Self::open_review(
922 repo,
923 &branch,
924 state,
925 roles,
926 max_parallel,
927 base_commit,
928 &takeover,
929 )
930 .await;
931 match opened {
932 Ok(runner) => Ok(runner),
933 Err(e) => {
934 if let Ok(mut partial) = RunState::load(&id) {
935 partial.status = RunStatus::Blocked;
936 partial.driver_exited = true;
937 partial.event(
938 "recovery",
939 format!("startup failed; retained source and candidate {branch}: {e:#}"),
940 );
941 partial.save()?;
942 }
943 Err(crate::pr_recovery::StartupFailed {
944 run: id,
945 cause: format!("{e:#}"),
946 }
947 .into())
948 }
949 }
950 }
951
952 async fn inherit(
957 repo: &Path,
958 branch: &str,
959 base_commit: &str,
960 takeover: &crate::handover::Takeover,
961 ) -> Option<crate::recovery::RecoveryRecord> {
962 use crate::recovery::{Relation, assess};
963 let newest = takeover.earlier.last()?;
964 let mut chosen = None;
965 for id in takeover.earlier.iter().rev() {
966 if let Ok(old) = RunState::load_under(id, &takeover.home)
967 && !old.reviews.is_empty()
968 && (old.winner().is_some_and(|w| w.branch == branch)
969 || old.pr_head.as_deref() == Some(branch))
970 {
971 chosen = Some(old);
972 break;
973 }
974 }
975 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976 .await
977 .unwrap_or_default();
978 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979 Some(last) if last.head == tip => Relation::Same,
980 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981 _ => Relation::Diverged,
982 };
983 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984 Some(assess(from, chosen.as_ref(), base_commit, relation))
985 }
986
987 async fn open_review(
990 repo: &Path,
991 branch: &str,
992 mut state: RunState,
993 roles: ResolvedRoles,
994 max_parallel: usize,
995 base_commit: String,
996 takeover: &crate::handover::Takeover,
997 ) -> Result<Self> {
998 let wait = repository_wait(&state, None);
999 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000 if state.pr_recovery.is_none() {
1001 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002 }
1003 let start = review_base(
1006 repo,
1007 &state.config.merge.remote,
1008 &state.base_branch,
1009 &base_commit,
1010 branch,
1011 )
1012 .await;
1013 let log = git::log_oneline(repo, &start, branch)
1014 .await
1015 .unwrap_or_default();
1016 let log_text = if log.trim().is_empty() {
1017 "(no commit messages)"
1018 } else {
1019 log.trim()
1020 };
1021 let mut instruction = format!(
1022 "Review the work already on branch `{branch}`. There is no task \
1023 statement: what the change claims to do is whatever its commits \
1024 say.\n\n{log_text}"
1025 );
1026 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030 if let Some(task) = takeover
1031 .task_instruction
1032 .as_deref()
1033 .filter(|t| !t.trim().is_empty())
1034 && rec.reviewed_head.is_some()
1035 {
1036 instruction = format!(
1037 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038 task.trim()
1039 );
1040 rec.criteria_inherited = true;
1041 }
1042 state.event("recovery", rec.summary());
1043 state.recovery = Some(rec);
1044 }
1045 if let Some(selected) = &state.pr_recovery {
1046 instruction = selected.instruction.clone();
1047 }
1048 state.task_instruction = takeover.task_instruction.clone();
1049 state.instruction = instruction;
1050 state.reviewed_commits = Some(
1051 git::subjects(repo, &start, branch)
1052 .await
1053 .unwrap_or_default(),
1054 );
1055
1056 for id in &takeover.earlier {
1059 if let Ok(old) = RunState::load_under(id, &takeover.home)
1060 && old.repo == state.repo
1061 && (old.pr_head.as_deref() == Some(branch)
1062 || old.winner().is_some_and(|w| w.branch == branch))
1063 {
1064 for hold in old.branch_owner_holds {
1065 if !state.branch_owner_holds.contains(&hold) {
1066 state.event(
1067 "approval-policy",
1068 format!(
1069 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070 hold.control.key(),
1071 hold.file,
1072 hold.head
1073 ),
1074 );
1075 state.branch_owner_holds.push(hold);
1076 }
1077 }
1078 }
1079 }
1080 let reviewed_head = git::rev_parse(repo, branch).await?;
1081 state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083 let worktree = state.worktree_root().join("under-review");
1086 if let Some(parent) = worktree.parent() {
1087 tokio::fs::create_dir_all(parent).await.ok();
1088 }
1089 let path = worktree.to_string_lossy().to_string();
1090 git::git(repo, &["worktree", "add", &path, branch])
1091 .await
1092 .with_context(|| {
1093 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094 })?;
1095
1096 let setup = match ensure_setup_config(&mut state, repo).await {
1097 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098 Err(e) => Err(e),
1099 };
1100 if let Err(e) = setup {
1101 git::worktree_remove_clean(repo, &worktree).await.ok();
1102 return Err(e);
1103 }
1104 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105 .await
1106 .unwrap_or(0);
1107 if commits == 0 {
1108 git::worktree_remove_clean(repo, &worktree).await.ok();
1109 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110 }
1111 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112 .await
1113 .map(|f| f.len())
1114 .unwrap_or(0);
1115 if files == 0
1116 && let (Ok(head_tree), Ok(base_tree)) = (
1117 git::tree_of(&worktree, "HEAD").await,
1118 git::tree_of(&worktree, &base_commit).await,
1119 )
1120 && head_tree == base_tree
1121 {
1122 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123 git::worktree_remove_clean(repo, &worktree).await.ok();
1124 bail!(
1125 "`{branch}` at {} has a tree identical to base {}; this usually means \
1126 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127 against `{}/{branch}`) rather than an empty change",
1128 short(&head),
1129 short(&base_commit),
1130 state.config.merge.remote
1131 );
1132 }
1133 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134 .await
1135 .unwrap_or_default();
1136
1137 state.candidates.push(Candidate {
1138 index: 0,
1139 label: 'A',
1140 agent: EXISTING_BRANCH.to_owned(),
1143 branch: branch.to_owned(),
1144 worktree,
1145 summary: String::new(),
1146 stat,
1147 files,
1148 commits,
1149 empty: false,
1150 failed: None,
1151 verified_noop: None,
1152 duration_ms: 0,
1153 folded: false,
1154 });
1155 state.tally = Some(Tally {
1156 first_choice: BTreeMap::from([('A', 0)]),
1157 borda: BTreeMap::new(),
1158 winner: 'A',
1159 rankings: 0,
1160 unanimous_initial: false,
1161 deliberated: false,
1162 changed_votes: 0,
1163 unanimous_final: false,
1164 tie_break: None,
1165 judges: 0,
1169 present: 0,
1170 quorum: 0,
1171 met_quorum: true,
1172 uncontested: Some("review-only run: nothing competed".to_owned()),
1173 });
1174 state.status = RunStatus::Reviewing;
1175 state.prep_complete = Some(true);
1176 state.event(
1177 "start",
1178 format!(
1179 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180 state.id
1181 ),
1182 );
1183 state.save()?;
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub fn resume(id: &str) -> Result<Self> {
1195 let state = RunState::load(id)?;
1196 crate::pr_head::ensure_owner(&state)?;
1197 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198 bail!(
1199 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200 );
1201 }
1202 if let Some(to) = &state.released_to {
1203 bail!(
1204 "run {} cannot be resumed: its worktree was released to run {}",
1205 state.short(),
1206 crate::run::short_of(to)
1207 );
1208 }
1209 let roles = state.config.resolve_roles()?;
1210 if let Some(id) = &state.config.graph.selected_implementer
1211 && state.candidates.iter().any(|c| c.agent != *id)
1212 {
1213 bail!(
1214 "replacement implementer pin does not match the saved candidates; task must stay held"
1215 );
1216 }
1217 let max_parallel = state.config.graph.max_parallel.max(1);
1218 Ok(Self {
1219 state,
1220 roles,
1221 sem: Arc::new(Semaphore::new(max_parallel)),
1222 pause: Pause::new(),
1223 interrupt: Pause::new(),
1224 })
1225 }
1226
1227 pub async fn execute(&mut self) -> Result<()> {
1234 if let Some(selection) = &self.state.pr_recovery
1235 && (selection.source_proof.is_some() || selection.source_run.is_some())
1236 {
1237 Box::pin(selection.verify_source(&self.state.repo)).await?;
1238 }
1239 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242 || self.state.land_approval.is_some()
1243 || self.state.merge_decision.is_some()
1244 || !self.state.merge_repairs.is_empty()
1245 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246 {
1247 let claim = FixClaim::acquire(&self.state.dir())?;
1248 let disk = RunState::load(&self.state.id)?;
1249 let checkpoint = |s: &RunState| {
1250 (
1251 s.merge_repairs.len(),
1252 s.merge_repairs
1253 .last()
1254 .map(|r| (r.question.clone(), r.phase)),
1255 s.land_approval
1256 .as_ref()
1257 .map(|a| (a.question.clone(), a.head.clone())),
1258 )
1259 };
1260 if checkpoint(&disk) != checkpoint(&self.state) {
1261 bail!(
1262 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263 );
1264 }
1265 Some(claim)
1266 } else {
1267 None
1268 };
1269 let mut execution = self.claim_driver().await?;
1270 let runner = &mut *execution.runner;
1271 let wait = runner.repository_wait();
1272 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275 let result = match result {
1276 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277 record_repository_block(&mut runner.state, &e);
1278 runner.park_here()?;
1279 Ok(())
1280 }
1281 other => other,
1282 };
1283 execution.driver.finish(&mut runner.state).await?;
1284 let ended = if result.is_err() {
1285 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1286 } else {
1287 crate::notices::run_ended(&runner.state)
1288 };
1289 if let Some(notice) = ended {
1290 crate::notices::raise(notice);
1291 }
1292 result
1293 }
1294
1295 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1296 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1297 }
1298
1299 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1300 let wait = self.repository_wait();
1301 coordinate_operation(&mut self.state, cwd, &wait).await
1302 }
1303
1304 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1306 let owner = match crate::pr_head::owner_lock_wait(
1307 &crate::run::home(),
1308 &self.state.id,
1309 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1310 )
1311 .await
1312 {
1313 Ok(owner) => owner,
1314 Err(error) => {
1315 let detail = format!("worktree maintenance or ownership update: {error:#}");
1316 self.state.resource_block = Some(crate::run::ResourceBlock {
1320 node: "start".into(),
1321 kind: "worktree maintenance".into(),
1322 detail: detail.clone(),
1323 at: Timestamp::now(),
1324 });
1325 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1326 }
1327 };
1328 let disk = RunState::load(&self.state.id)?;
1329 crate::pr_head::ensure_owner(&disk)?;
1330 if disk.liveness(false) == crate::run::Liveness::Live {
1331 bail!("run already has a live driver; wait for its execution to end");
1332 }
1333 if let Some(to) = &disk.released_to {
1334 bail!(
1335 "run {} cannot continue: its worktree was released to run {}",
1336 self.state.short(),
1337 crate::run::short_of(to)
1338 );
1339 }
1340 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1341 let mut execution = DriverExecution {
1342 runner: self,
1343 driver,
1344 };
1345 let _owner = owner;
1348 let state = &mut execution.runner.state;
1349 state.parked = false;
1354 state.resource_wait = None;
1355 state.resource_block = None;
1356 state.clear_active();
1363 let pid = std::process::id();
1378 state.driver_pid = Some(pid);
1379 state.driver_started_at = crate::proc::process_started_at(pid);
1380 state.driver_exited = false;
1381 state.begin_pass();
1382 execution.driver.bind(state);
1383 state.save()?;
1384 execution.driver.record(state)?;
1385 Ok(execution)
1386 }
1387
1388 async fn execute_graph(&mut self) -> Result<()> {
1389 self.state.configure_recovery_resume()?;
1390 self.persist_candidates().await?;
1393 if self
1394 .state
1395 .merge_decision
1396 .as_ref()
1397 .is_some_and(|decision| !decision.closed)
1398 && self.state.status != RunStatus::Landing
1399 {
1400 if crate::merge_decision::resumable_hold(&self.state) {
1401 self.state.merge = Some(MergeOutcome {
1402 mode: MergeMode::Pr,
1403 ok: false,
1404 detail: self
1405 .state
1406 .merge_decision
1407 .as_ref()
1408 .unwrap()
1409 .target
1410 .pr
1411 .clone(),
1412 empty: false,
1413 });
1414 self.run_land().await?;
1415 self.settle_questions();
1416 return Ok(());
1417 }
1418 bail!(
1419 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1420 );
1421 }
1422 if self.state.config.graph.selected_implementer.is_some()
1423 && self.state.status == RunStatus::Blocked
1424 && self.state.tally.is_none()
1425 && self.state.candidate_preservation.is_empty()
1426 && self.state.interrupted_implementations.is_empty()
1427 {
1428 bail!(
1429 "selected implementation is held; explicitly release the owning task to retry it"
1430 );
1431 }
1432
1433 if self.state.merge_repairs.last().is_some_and(|r| {
1434 matches!(
1435 r.phase,
1436 crate::run::MergeRepairPhase::Requested
1437 | crate::run::MergeRepairPhase::Fixing
1438 | crate::run::MergeRepairPhase::Reviewing
1439 )
1440 }) {
1441 if self.repair_merge_review().await? {
1442 self.run_land().await?;
1443 }
1444 self.settle_questions();
1445 return Ok(());
1446 }
1447 if self.state.pr_publication_pending {
1450 self.merge().await?;
1451 if self.resource_halt()? {
1452 return Ok(());
1453 }
1454 self.state.save()?;
1455 self.settle_questions();
1456 return Ok(());
1457 }
1458 if self.state.status == RunStatus::Stalled {
1471 if self.recover_stall().await? {
1472 self.finish_after_tally().await?;
1473 } else {
1474 self.state.save()?;
1476 }
1477 return Ok(());
1478 }
1479 if self.state.status == RunStatus::Landing
1489 || crate::handover::landing_checkpoint(&self.state)
1490 {
1491 self.state.land_sync_pending = true;
1492 self.state.save()?;
1493 self.run_land().await?;
1494 self.settle_questions();
1499 return Ok(());
1500 }
1501 if self.state.github_text.is_some()
1506 && self.state.merge.is_none()
1507 && self.state.status == RunStatus::Gating
1508 {
1509 self.merge().await?;
1510 if self.resource_halt()? {
1511 return Ok(());
1512 }
1513 self.state.save()?;
1514 return Ok(());
1515 }
1516 if let Err(e) = self.prep().await {
1517 if e.is::<crate::repo_lock::ResourceBlocked>() {
1518 return Err(e);
1519 }
1520 self.state.status = RunStatus::Blocked;
1521 self.state
1522 .prep_failure
1523 .get_or_insert_with(|| format!("{e:#}"));
1524 self.state
1525 .event("prep", format!("preparation stopped: {e:#}"));
1526 self.state.save()?;
1527 return Err(e);
1528 }
1529 if self.park_here()? {
1530 return Ok(());
1531 }
1532 self.advise().await?;
1533 if self.park_here()? {
1534 return Ok(());
1535 }
1536 self.implement().await?;
1537 if self.park_here()? {
1538 return Ok(());
1539 }
1540 if self.state.status == RunStatus::VerifiedNoop {
1544 return Ok(());
1545 }
1546 self.judge().await?;
1547 if self.park_here()? {
1548 return Ok(());
1549 }
1550 self.deliberate().await?;
1551 if self.park_here()? {
1552 return Ok(());
1553 }
1554 self.vote().await?;
1555 if self.park_here()? {
1556 return Ok(());
1557 }
1558 self.tally()?;
1559 if self.state.status == RunStatus::Stalled {
1564 self.state.save()?;
1568 return Ok(());
1569 }
1570 self.finish_after_tally().await?;
1571 Ok(())
1572 }
1573
1574 fn stop_for_resource(&mut self) -> Result<()> {
1579 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1581 {
1582 self.state.status = RunStatus::Blocked;
1583 }
1584 self.state.save()
1585 }
1586
1587 fn resource_halt(&mut self) -> Result<bool> {
1589 if self.state.resource_block.is_none() {
1590 return Ok(false);
1591 }
1592 self.stop_for_resource()?;
1593 Ok(true)
1594 }
1595
1596 fn park_here(&mut self) -> Result<bool> {
1603 let parking = self.pause.parked() || self.interrupt.parked();
1604 if !parking && self.state.resource_block.is_some() {
1605 self.stop_for_resource()?;
1606 return Ok(true);
1607 }
1608 if !self.pause.parked() && !self.interrupt.parked() {
1613 return Ok(false);
1614 }
1615 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1616 let why = match cause.as_ref().map(ParkCause::label) {
1617 Some(reason) => format!(
1618 "parked after `{}` ({reason}) — resume to carry on from here",
1619 self.state.status.as_str()
1620 ),
1621 None => format!(
1622 "parked after `{}` — resume to carry on from here",
1623 self.state.status.as_str()
1624 ),
1625 };
1626 self.state.event("park", why);
1627 self.state
1628 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1629 self.state.parked = true;
1630 self.state.save()?;
1631 Ok(true)
1632 }
1633
1634 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1636 if self.state.prep_failure.is_none() {
1637 return Ok(());
1638 }
1639 let failed_at = self
1640 .state
1641 .events
1642 .iter()
1643 .rev()
1644 .find(|event| event.node == "prep")
1645 .map_or(self.state.updated_at, |event| event.at);
1646 anyhow::ensure!(
1647 approval_at > failed_at,
1648 "preparation retry requires a new explicit release after the failure"
1649 );
1650 self.state.prep_failure = None;
1651 self.state.prep_complete = Some(false);
1652 self.state
1653 .event("prep-retry", "operator approved retry of preparation");
1654 Ok(())
1655 }
1656
1657 pub fn on_pause(&mut self, pause: Pause) {
1659 self.pause = pause;
1660 }
1661
1662 pub fn watch_interrupt(&mut self, pause: Pause) {
1668 self.interrupt = pause;
1669 }
1670
1671 fn settle_questions(&mut self) {
1691 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1692 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1693 }
1694 }
1695
1696 async fn finish_after_tally(&mut self) -> Result<()> {
1699 self.graft().await?;
1702 if self.resource_halt()? {
1703 return Ok(());
1704 }
1705 self.fold_losers().await?;
1706 self.sync_to_base().await?;
1711 if self.resource_halt()? {
1712 return Ok(());
1713 }
1714 if self.state.status == RunStatus::AlreadyInBase {
1715 return Ok(());
1716 }
1717 self.review_loop().await?;
1718 if self.resource_halt()? {
1719 return Ok(());
1720 }
1721 self.sync_to_base().await?;
1722 if self.resource_halt()? {
1723 return Ok(());
1724 }
1725 if self.state.status == RunStatus::AlreadyInBase {
1726 return Ok(());
1727 }
1728 self.gate().await?;
1729 self.merge().await?;
1730 if self.resource_halt()? {
1731 return Ok(());
1732 }
1733 self.state.save()?;
1734 Ok(())
1735 }
1736
1737 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1744 let result = match ensure_setup_config(&mut self.state, repo).await {
1745 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1746 Err(e) => Err(e),
1747 };
1748 if let Err(e) = result {
1749 git::worktree_remove_clean(repo, wt).await.ok();
1750 if in_prep {
1751 self.rollback_prep(repo).await;
1752 }
1753 self.state.status = RunStatus::Blocked;
1754 self.state.event(
1755 "setup",
1756 format!("worktree setup failed in {}: {e:#}", wt.display()),
1757 );
1758 self.state.save()?;
1759 return Err(e);
1760 }
1761 Ok(())
1762 }
1763
1764 async fn rollback_prep(&mut self, repo: &Path) {
1766 let root = self.state.worktree_root();
1767 for c in std::mem::take(&mut self.state.candidates) {
1768 if matches!(
1769 git::worktree_remove_clean(repo, &c.worktree).await,
1770 Ok(true)
1771 ) {
1772 git::branch_delete(repo, &c.branch).await.ok();
1773 } else {
1774 self.state.candidates.push(c);
1775 }
1776 }
1777 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1778 for j in 1..=seats {
1779 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1780 .await
1781 .ok();
1782 }
1783 for k in 1..=self.state.config.graph.advisors {
1784 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1785 .await
1786 .ok();
1787 }
1788 }
1789
1790 async fn prep(&mut self) -> Result<()> {
1791 if let Some(reason) = &self.state.prep_failure {
1792 bail!(
1793 "candidate preparation previously failed: {reason}; correct the source before retrying"
1794 );
1795 }
1796 if self
1797 .state
1798 .prep_complete
1799 .unwrap_or(!self.state.candidates.is_empty())
1800 {
1801 return Ok(());
1802 }
1803 let repo = self.state.repo.clone();
1804 let _coordination = self.coordinate(&repo).await?;
1807 let base = self.state.base_commit.clone();
1808 let plan = self.state.sources.plan(&repo, &base).await?;
1809 self.rollback_prep(&repo).await;
1810 if !self.state.candidates.is_empty() {
1811 bail!(
1812 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1813 );
1814 }
1815 self.state.prep_complete = Some(false);
1816 self.state.status = RunStatus::Prep;
1817 self.state.save()?;
1818 let repo = self.state.repo.clone();
1819 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1820 let root = self.state.worktree_root();
1821 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1822
1823 let hooks_dir = self.state.dir().join("hooks");
1826 if self.state.config.blind.commit_msg_hook {
1827 std::fs::create_dir_all(&hooks_dir)
1828 .with_context(|| format!("create {}", hooks_dir.display()))?;
1829 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1830 let path = hooks_dir.join("commit-msg");
1831 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1832 make_executable(&path)?;
1833 git::acquire_worktree_config(&repo).await?;
1841 self.state.enabled_worktree_config = true;
1842 }
1843
1844 ensure_setup_config(&mut self.state, &repo).await?;
1847
1848 for (index, (spec, label)) in self
1849 .roles
1850 .implementers
1851 .clone()
1852 .into_iter()
1853 .zip(labels)
1854 .enumerate()
1855 {
1856 let branch = self.state.branch_for(label);
1857 let worktree = root.join(format!("cand-{label}"));
1858 self.state.event(
1862 "prep-intent",
1863 serde_json::json!({
1864 "branch": branch,
1865 "worktree": worktree,
1866 "start": start,
1867 "picks": plan.picks,
1868 })
1869 .to_string(),
1870 );
1871 self.state.save()?;
1872 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1873 if self.state.config.blind.commit_msg_hook {
1874 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1875 }
1876 git::local_exclude(&worktree, "/.magi/").await?;
1877 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1878 git::worktree_remove_clean(&repo, &worktree).await.ok();
1881 git::branch_delete(&repo, &branch).await.ok();
1882 self.rollback_prep(&repo).await;
1883 self.state.status = RunStatus::Blocked;
1884 self.state
1885 .event("prep", format!("worktree setup failed: {e:#}"));
1886 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1887 self.state.save()?;
1888 return Err(e);
1889 }
1890 self.state.candidates.push(Candidate {
1891 index,
1892 label,
1893 agent: spec.id.clone(),
1894 branch,
1895 worktree,
1896 summary: String::new(),
1897 stat: String::new(),
1898 files: 0,
1899 commits: 0,
1900 empty: false,
1901 failed: None,
1902 verified_noop: None,
1903 duration_ms: 0,
1904 folded: false,
1905 });
1906 self.state.save()?;
1907 for pick in &plan.picks {
1908 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1909 {
1910 self.state.status = RunStatus::Blocked;
1911 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1912 self.state
1913 .event("prep", format!("cannot apply referenced commit: {e}"));
1914 self.state.save()?;
1915 return Err(e);
1916 }
1917 }
1918 }
1919
1920 for j in 1..=self.roles.judges.len() {
1921 let wt = root.join(format!("judge-{j}"));
1922 if !wt.exists() {
1923 git::worktree_add_detached(&repo, &wt, &base).await?;
1924 self.setup_seat_worktree(&repo, &wt, true).await?;
1925 }
1926 }
1927
1928 if self.state.config.graph.advise {
1936 for k in 1..=self.state.config.graph.advisors {
1937 let wt = root.join(format!("advisor-{k}"));
1938 if !wt.exists() {
1939 git::worktree_add_detached(&repo, &wt, &base).await?;
1940 self.setup_seat_worktree(&repo, &wt, true).await?;
1941 }
1942 }
1943 }
1944
1945 let authors: Vec<&str> = self
1950 .roles
1951 .implementers
1952 .iter()
1953 .map(|a| a.id.as_str())
1954 .collect();
1955 let overlap: Vec<String> = self
1956 .roles
1957 .judges
1958 .iter()
1959 .enumerate()
1960 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1961 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1962 .collect();
1963 if !overlap.is_empty() {
1964 let note = format!(
1965 "{} also authored a candidate; blind, but the panel is less \
1966 independent than {} distinct agents would be",
1967 overlap.join(", "),
1968 self.roles.judges.len()
1969 );
1970 self.state.event("prep", note);
1971 }
1972
1973 self.state.event(
1974 "prep",
1975 format!(
1976 "{} candidates, {} judges, base {} ({})",
1977 self.state.candidates.len(),
1978 self.roles.judges.len(),
1979 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1980 self.state.base_branch
1981 ),
1982 );
1983 self.state.prep_complete = Some(true);
1984 self.state.status = RunStatus::Implementing;
1985 self.state.save()?;
1986 Ok(())
1987 }
1988
1989 async fn advise(&mut self) -> Result<()> {
2022 let implement_untouched = self
2023 .state
2024 .candidates
2025 .iter()
2026 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2027 if !self.state.config.graph.advise || self.state.advise_attempted {
2028 return Ok(());
2029 }
2030 if !implement_untouched {
2031 self.state.event(
2032 "advise",
2033 "skipping the design-deliberation stage: at least one \
2034 candidate already shows implementation progress, so this \
2035 run is past the point the stage exists to run before"
2036 .to_owned(),
2037 );
2038 self.state.advise_attempted = true;
2039 self.state.save()?;
2040 return Ok(());
2041 }
2042 let run_id = self.state.id.clone();
2043 let prompts = self.state.config.prompts.clone();
2044 let instruction = self.state.instruction.clone();
2045 let language = self.state.config.graph.language.clone();
2046 let root = self.state.worktree_root();
2047 let n = self.state.config.graph.advisors;
2048 let where_recorded = self.state.dir().join("run.json");
2049
2050 let seats = match self.state.config.advisors() {
2051 Ok(seats) if !seats.is_empty() => seats,
2052 Ok(_) => {
2053 self.state.event(
2054 "advise",
2055 format!(
2056 "[graph] advisors is 0; skipping the design-deliberation \
2057 stage and continuing without a synthesis brief (see {})",
2058 where_recorded.display()
2059 ),
2060 );
2061 self.state.advise_attempted = true;
2062 self.state.save()?;
2063 return Ok(());
2064 }
2065 Err(e) => {
2066 self.state.event(
2067 "advise",
2068 format!(
2069 "could not resolve advisor seats ({e:#}); continuing \
2070 without a design-deliberation brief (see {})",
2071 where_recorded.display()
2072 ),
2073 );
2074 self.state.advise_attempted = true;
2075 self.state.save()?;
2076 return Ok(());
2077 }
2078 };
2079
2080 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2081 let artifacts = agent::artifacts_dir(&self.state.dir());
2082 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2083
2084 let mut jobs = Vec::new();
2085 for (i, spec) in seats.iter().cloned().enumerate() {
2086 let seat_key = format!("advisor-{}", i + 1);
2087 let seat = self.seat(&seat_key, &spec.id);
2088 jobs.push(SeatJob {
2089 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2090 spec,
2091 seat,
2092 cwd: worktrees[i % worktrees.len()].clone(),
2093 timeout,
2094 allow_write: false,
2095 sessions: false,
2096 artifacts: artifacts.clone(),
2097 stem: seat_key,
2098 handover: None,
2099 });
2100 }
2101
2102 self.state.event(
2103 "advise",
2104 format!(
2105 "{} advisor seat(s) sketching a design in parallel",
2106 jobs.len()
2107 ),
2108 );
2109 let mut quota_losses = Vec::new();
2110 let cache = self.state.config.cache_dir();
2111 let ctx = WaveCtx {
2112 carry_seats: false,
2113 pauses: Some((&self.pause, &self.interrupt)),
2114 run: &run_id,
2115 node: "advise",
2116 prompts: &prompts,
2117 cache: cache.as_deref(),
2118 round: None,
2119 };
2120 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2121 let results = ask_json_wave::<Proposal>(
2122 jobs,
2123 Arc::clone(&self.sem),
2124 self.state.config.graph.retries,
2125 &advisor_roster,
2126 &ctx,
2127 &mut quota_losses,
2128 &mut self.state,
2129 &|p: &Proposal| p.validate(),
2130 )
2131 .await;
2132 if self.state.resource_block.is_some() {
2133 return self.stop_for_resource();
2134 }
2135 self.state.quota.extend(quota_losses);
2136
2137 let mut records = Vec::with_capacity(results.len());
2138 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2139 let agent_id = seat.agent.clone();
2140 self.state.seats.insert(seat.key.clone(), seat);
2141 match res {
2142 Ok((proposal, out)) => {
2143 self.state
2144 .event("advise", format!("advisor-{} proposed a design", i + 1));
2145 records.push(advise::AdvisorRecord::proposed(
2146 i + 1,
2147 agent_id,
2148 proposal,
2149 out.duration_ms,
2150 ));
2151 }
2152 Err(e) => {
2153 self.state.event(
2154 "advise",
2155 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2156 );
2157 records.push(advise::AdvisorRecord::failed(
2158 i + 1,
2159 agent_id,
2160 e.to_string(),
2161 ));
2162 }
2163 }
2164 }
2165
2166 let mut advice = advise::Advice {
2167 records,
2168 synthesis: None,
2169 };
2170 if advice.proposals().is_empty() {
2171 self.state.event(
2172 "advise",
2173 "no advisor produced a usable proposal; continuing without a \
2174 synthesis brief"
2175 .to_owned(),
2176 );
2177 } else {
2178 match self
2179 .synthesize_brief(
2180 &advice,
2181 &instruction,
2182 &language,
2183 &worktrees[0],
2184 &artifacts,
2185 &run_id,
2186 &prompts,
2187 cache.as_deref(),
2188 )
2189 .await
2190 {
2191 Ok(Some(text)) => {
2192 self.state.event(
2193 "advise",
2194 "synthesized a design brief for the implementer".to_owned(),
2195 );
2196 advice.synthesis = Some(text);
2197 }
2198 Ok(None) => {
2199 self.state.event(
2200 "advise",
2201 "the synthesis seat produced nothing usable; continuing \
2202 without a design brief"
2203 .to_owned(),
2204 );
2205 }
2206 Err(e) => {
2207 if e.is::<crate::repo_lock::ResourceBlocked>() {
2208 return Err(e);
2209 }
2210 self.state.event(
2211 "advise",
2212 format!("could not synthesize a design brief: {e:#}"),
2213 );
2214 }
2215 }
2216 }
2217 advise::apply_reflection(&mut advice);
2218
2219 self.state.advice = Some(advice);
2220 self.state.advise_attempted = true;
2221 self.state.save()?;
2222 Ok(())
2223 }
2224
2225 #[allow(clippy::too_many_arguments)]
2237 async fn synthesize_brief(
2238 &mut self,
2239 advice: &advise::Advice,
2240 instruction: &str,
2241 language: &str,
2242 cwd: &Path,
2243 artifacts: &Path,
2244 run_id: &str,
2245 prompts: &Prompts,
2246 cache: Option<&Path>,
2247 ) -> Result<Option<String>> {
2248 let chain = agent::pick_chain(
2249 &self.state.config.agents,
2250 self.state.config.roles.synthesizer.as_ref(),
2251 &agent::installed,
2252 "synthesizer",
2253 )?;
2254 let proposals = advice.proposals();
2255 let mut prompt = prompt::with_overlay(
2256 prompt::synthesize_brief(instruction, &proposals, language),
2257 prompts.overlay("advise"),
2258 );
2259 if cache.is_some() {
2260 prompt.push('\n');
2265 prompt.push_str(&prompt::build_cache_note("advise", false));
2266 }
2267 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2268 let mut last = None;
2273 for (n, spec) in chain.iter().enumerate() {
2274 if n > 0 {
2275 self.state
2276 .event("advise", format!("synthesis falling back to {}", spec.id));
2277 }
2278 let mut seat = self.seat("advise-synthesis", &spec.id);
2279 let wait = self.repository_wait();
2280 let outcome = wait
2281 .scope(agent::invoke(
2282 spec,
2283 &mut seat,
2284 &Invocation {
2285 scope: crate::agent::InvocationScope::Repository,
2286 coordination: self.state.config.coordination.as_ref(),
2287 fetched_base: None,
2288 cwd,
2289 prompt: &prompt,
2290 timeout,
2291 allow_write: false,
2292 unsandboxed: false,
2293 sessions: false,
2294 artifacts,
2295 stem: &if n == 0 {
2296 "advise-synthesis".to_owned()
2297 } else {
2298 format!("advise-synthesis-{}", spec.id)
2299 },
2300 run: run_id,
2301 node: "advise",
2302 cache_dir: None,
2303 attachments: &[],
2304 writable: &[],
2305 },
2306 ))
2307 .await;
2308 if outcome.is_ok() {
2309 self.state.seats.insert(seat.key.clone(), seat);
2310 }
2311 let advance = agent::chain_advances(&outcome);
2312 last = Some(outcome);
2313 if !advance {
2314 break;
2315 }
2316 }
2317 let out = last.expect("a chain holds at least one agent")?;
2320 if !out.usable() {
2321 return Ok(None);
2322 }
2323 let text =
2324 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2325 Ok((!text.trim().is_empty()).then_some(text))
2326 }
2327
2328 async fn implement(&mut self) -> Result<()> {
2331 self.persist_candidates().await?;
2332 let run_id = self.state.id.clone();
2337 let prompts = self.state.config.prompts.clone();
2338 let todo: Vec<usize> = self
2339 .state
2340 .candidates
2341 .iter()
2342 .enumerate()
2343 .filter(|(_, c)| {
2344 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2345 && c.failed.is_none()
2346 && !c.empty
2347 })
2348 .map(|(i, _)| i)
2349 .collect();
2350 if todo.is_empty() {
2351 return self.after_implement();
2352 }
2353 self.state.status = RunStatus::Implementing;
2354
2355 let language = self.state.config.graph.language.clone();
2356 let github_language = self.state.config.graph.github_language.clone();
2357 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2358 let sessions = self.state.config.graph.sessions;
2359 let artifacts = agent::artifacts_dir(&self.state.dir());
2360 let brief = self
2364 .state
2365 .advice
2366 .as_ref()
2367 .and_then(|a| a.synthesis.as_deref())
2368 .map(str::to_owned);
2369 let attachments = self.state.attachments.clone();
2370
2371 let mut jobs = Vec::new();
2372 for &i in &todo {
2373 let (index, label, worktree) = {
2374 let c = &self.state.candidates[i];
2375 (c.index, c.label, c.worktree.clone())
2376 };
2377 let spec = self.roles.implementers[index].clone();
2378 let seat_key = format!("impl-{label}");
2379 let seat = self.seat(&seat_key, &spec.id);
2380 let instruction = seeded_instruction(&self.state);
2381 jobs.push(SeatJob {
2382 spec,
2383 seat,
2384 prompt: prompt::implement_in(
2385 &instruction,
2386 &worktree.to_string_lossy(),
2387 &language,
2388 &github_language,
2389 brief.as_deref(),
2390 &attachments,
2391 ),
2392 cwd: worktree,
2393 timeout,
2394 allow_write: true,
2395 sessions,
2396 artifacts: artifacts.clone(),
2397 stem: format!("impl-{label}"),
2398 handover: None,
2399 });
2400 }
2401
2402 self.state.event(
2403 "implement",
2404 format!("{} candidates in parallel", jobs.len()),
2405 );
2406 let mut sent = jobs.clone();
2412 let cache = self.state.config.cache_dir();
2413 let ctx = WaveCtx {
2414 carry_seats: false,
2415 pauses: Some((&self.pause, &self.interrupt)),
2416 run: &run_id,
2417 node: "implement",
2418 prompts: &prompts,
2419 cache: cache.as_deref(),
2420 round: None,
2421 };
2422 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2423 if !wave_blocked(&results) {
2424 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2425 .await;
2426 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2427 .await?;
2428 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2429 .await;
2430 }
2431 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2432
2433 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2434 && results
2435 .iter()
2436 .any(|(_, _, out)| FailClass::of(out).is_some());
2437 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2438 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2439 if self
2443 .state
2444 .candidate_preservation
2445 .contains_key(&self.state.candidates[i].label)
2446 {
2447 self.state.seats.insert(seat.key.clone(), seat);
2448 }
2449 continue;
2450 }
2451 let seat_key = seat.key.clone();
2452 let agent = seat.agent.clone();
2461 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2462 self.state.seats.insert(seat.key.clone(), seat);
2463 let label = self.state.candidates[i].label;
2464 self.state.interrupted_implementations.remove(&label);
2465
2466 let (summary, duration, failed, verified_claim) = match out {
2467 AgentOutcome::Ok(o) => {
2468 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2469 let failed = (!o.usable()).then(|| {
2470 if o.timed_out {
2471 "agent timed out".to_owned()
2472 } else {
2473 format!(
2474 "agent exited with {}",
2475 crate::notices::exit_status(o.exit_code)
2476 )
2477 }
2478 });
2479 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2480 (text, o.duration_ms, failed, verified_claim)
2481 }
2482 AgentOutcome::Dropped(o) => {
2488 let why = o
2489 .dropped
2490 .as_ref()
2491 .map(|d| d.why.as_str())
2492 .unwrap_or("the CLI ended the stream without delivering its answer");
2493 (
2494 String::new(),
2495 o.duration_ms,
2496 Some(format!("the CLI dropped the stream ({why})")),
2497 None,
2498 )
2499 }
2500 AgentOutcome::Quota(o) => {
2501 self.state.quota.push(QuotaLoss {
2502 seat: seat_key,
2503 node: "implement".to_owned(),
2504 at: Timestamp::now(),
2505 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2506 });
2507 (
2508 String::new(),
2509 o.duration_ms,
2510 Some("rate limited (quota); produced no change".to_owned()),
2511 None,
2512 )
2513 }
2514 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2515 (String::new(), 0, Some(e), None)
2516 }
2517 };
2518
2519 let c = &mut self.state.candidates[i];
2522 if !exhausted_the_fallback_chain {
2523 c.agent = agent;
2524 }
2525 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2526 c.duration_ms = duration;
2527 c.failed = failed;
2528 c.verified_noop = verified_claim;
2529 self.state
2530 .candidate_preservation
2531 .insert(label, "awaiting persistence".to_owned());
2532 }
2533 self.state.save()?;
2534 self.persist_candidates().await?;
2535
2536 if resource_blocked {
2537 return self.stop_for_resource();
2538 }
2539 if selected_failed {
2540 self.state.status = RunStatus::Blocked;
2541 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2542 self.state.save()?;
2543 bail!(
2544 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2545 );
2546 }
2547 self.after_implement()
2548 }
2549
2550 async fn persist_candidates(&mut self) -> Result<()> {
2552 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2553 for label in labels {
2554 let i = self
2555 .state
2556 .candidates
2557 .iter()
2558 .position(|c| c.label == label)
2559 .context("pending candidate is missing")?;
2560 let c = self.state.candidates[i].clone();
2561 let interrupted = self.state.interrupted_implementations.contains(&label);
2562 let result = async {
2563 let wait = crate::repo_lock::WaitPolicy::new(
2564 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2565 || false,
2566 );
2567 let _coordination =
2568 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2569 if let Err(e) = git::snapshot_uncommitted(
2572 &c.worktree,
2573 &self
2574 .state
2575 .dir()
2576 .join("artifacts")
2577 .join(format!("candidate-{label}-recovery")),
2578 )
2579 .await
2580 {
2581 self.state.event(
2582 "preserve",
2583 format!("candidate {label} snapshot incomplete: {e:#}"),
2584 );
2585 }
2586 let rescued = git::rescue_commit(
2587 &c.worktree,
2588 &format!("magi: candidate {label} (uncommitted work)"),
2589 )
2590 .await?;
2591 self.state.note_withheld("implement", &rescued.withheld);
2592 let base = &self.state.base_commit;
2593 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2594 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2595 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2596 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2597 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2598 if c.empty && commits > 0 && !patch.trim().is_empty()
2599 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2600 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2601 || self.state.tally.is_some())
2602 {
2603 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2607 }
2608 let c = &mut self.state.candidates[i];
2609 c.commits = commits;
2610 c.stat = stat;
2611 c.files = files;
2612 let no_changes = commits == 0 || patch.trim().is_empty();
2613 c.empty = no_changes && !interrupted;
2614 if interrupted && no_changes {
2615 c.failed = None;
2616 c.verified_noop = None;
2617 }
2618 if !c.empty {
2619 c.failed = None;
2620 c.verified_noop = None;
2621 }
2622 self.state.event(
2623 "preserve",
2624 format!("candidate {label}: {files} files, {commits} commits"),
2625 );
2626 Ok::<_, anyhow::Error>(())
2627 }
2628 .await;
2629 match result {
2630 Ok(()) => {
2631 self.state.candidate_preservation.remove(&label);
2632 if !interrupted || self.state.candidates[i].files > 0 {
2633 self.state.interrupted_implementations.remove(&label);
2634 }
2635 }
2636 Err(e) => {
2637 let reason = format!("candidate {label} preservation failed: {e:#}");
2638 self.state.event("preserve", &reason);
2639 self.state.candidate_preservation.insert(label, reason);
2640 }
2641 }
2642 if self.state.candidate_preservation.is_empty()
2643 && self.state.status == RunStatus::Blocked
2644 && self.state.candidates.iter().all(|c| c.failed.is_none())
2645 {
2646 self.state.status = RunStatus::Implementing;
2650 self.state.event(
2651 "preserve",
2652 "candidate persistence recovered; implementation can continue",
2653 );
2654 }
2655 self.state.save()?;
2656 }
2657 if !self.state.candidate_preservation.is_empty() {
2658 self.state.status = RunStatus::Blocked;
2659 self.state.save()?;
2660 bail!(
2661 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2662 );
2663 }
2664 Ok(())
2665 }
2666
2667 async fn resume_undelivered(
2695 &mut self,
2696 results: &mut [(usize, SeatState, AgentOutcome)],
2697 sent: &[SeatJob],
2698 prompts: &Prompts,
2699 run_id: &str,
2700 ) {
2701 for (wi, seat, out) in results.iter_mut() {
2702 let Some(dropped) = (match &*out {
2703 AgentOutcome::Dropped(o) => o.dropped.clone(),
2704 _ => None,
2705 }) else {
2706 continue;
2707 };
2708 let Some(job) = sent.get(*wi) else { continue };
2709 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2711 self.state.event(
2712 "implement",
2713 format!(
2714 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2715 work is in the tree",
2716 seat.key, dropped.output_tokens, dropped.why
2717 ),
2718 );
2719 continue;
2720 }
2721 if !has_context(&job.spec, seat, job.sessions) {
2729 self.state.event(
2730 "implement",
2731 format!(
2732 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2733 is no session left to resume",
2734 seat.key, dropped.output_tokens, dropped.why
2735 ),
2736 );
2737 continue;
2738 }
2739 self.state.event(
2740 "implement",
2741 format!(
2742 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2743 conversation",
2744 seat.key, dropped.output_tokens, dropped.why
2745 ),
2746 );
2747 let mut retry = job.clone();
2748 retry.seat = seat.clone();
2749 retry.prompt = prompt::resume_after_drop(&dropped.why);
2750 retry.timeout = retry_budget(job.timeout, true);
2751 retry.stem = format!("{}-resume", job.stem);
2752 let cache = self.state.config.cache_dir();
2753 let ctx = WaveCtx {
2754 carry_seats: false,
2755 pauses: Some((&self.pause, &self.interrupt)),
2756 run: run_id,
2757 node: "implement",
2758 prompts,
2759 cache: cache.as_deref(),
2760 round: None,
2761 };
2762 let (resumed_seat, resumed) =
2763 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2764 *seat = resumed_seat;
2765 *out = resumed;
2766 }
2767 }
2768
2769 async fn resume_seat_handovers(
2834 &mut self,
2835 results: &mut [(usize, SeatState, AgentOutcome)],
2836 sent: &mut [SeatJob],
2837 prompts: &Prompts,
2838 run_id: &str,
2839 ) -> Result<()> {
2840 let instruction = seeded_instruction(&self.state);
2841 let language = self.state.config.graph.language.clone();
2842 let github_language = self.state.config.graph.github_language.clone();
2843 let brief = self
2844 .state
2845 .advice
2846 .as_ref()
2847 .and_then(|a| a.synthesis.as_deref())
2848 .map(str::to_owned);
2849 let attachments = self.state.attachments.clone();
2850 for (wi, seat, out) in results.iter_mut() {
2851 let others: BTreeSet<String> = sent
2854 .iter()
2855 .enumerate()
2856 .filter(|(j, _)| j != wi)
2857 .map(|(_, j)| j.spec.id.clone())
2858 .collect();
2859 let Some(job) = sent.get_mut(*wi) else {
2860 continue;
2861 };
2862 let start = self
2867 .roles
2868 .implementer_roster
2869 .iter()
2870 .position(|s| s.id == job.spec.id)
2871 .unwrap_or(0);
2872 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2873 let mut fallback_attempt = 0usize;
2874 let mut prev: Option<FailClass> = None;
2875 while let Some(cur) = FailClass::of(&*out) {
2876 if !should_hand_over(prev.as_ref(), &cur) {
2877 break;
2878 }
2879 let Some(next) =
2880 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2881 .cloned()
2882 else {
2883 break;
2884 };
2885 tried.insert(next.id.clone());
2886 fallback_attempt += 1;
2887
2888 let coordination = self.coordinate(&job.cwd).await?;
2889 if let Ok(r) = git::rescue_commit(
2890 &job.cwd,
2891 &format!(
2892 "magi: candidate {} (uncommitted work before {} fallback)",
2893 seat.key,
2894 if cur == FailClass::Quota {
2895 "quota"
2896 } else {
2897 "handover"
2898 }
2899 ),
2900 )
2901 .await
2902 {
2903 self.state.note_withheld("implement", &r.withheld);
2904 }
2905 drop(coordination);
2906
2907 record_handover(
2908 &mut self.state,
2909 "implement",
2910 &seat.key,
2911 &seat.agent,
2912 &next.id,
2913 &cur,
2914 &fail_reason(&*out),
2915 );
2916 prev = Some(cur.clone());
2917
2918 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2919 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2920 job.spec = next.clone();
2928 let mut retry = job.clone();
2929 retry.seat = new_seat;
2930 retry.prompt = prompt::implement_in(
2931 &instruction,
2932 &job.cwd.to_string_lossy(),
2933 &language,
2934 &github_language,
2935 brief.as_deref(),
2936 &attachments,
2937 );
2938 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2939 let cache = self.state.config.cache_dir();
2940 let ctx = WaveCtx {
2941 carry_seats: false,
2942 pauses: Some((&self.pause, &self.interrupt)),
2943 run: run_id,
2944 node: "implement",
2945 prompts,
2946 cache: cache.as_deref(),
2947 round: None,
2948 };
2949 let (fallback_seat, fallback_out) = run_one(
2950 retry,
2951 Arc::clone(&self.sem),
2952 &ctx,
2953 &mut self.state,
2954 fallback_attempt,
2955 )
2956 .await;
2957 *seat = fallback_seat;
2958 *out = fallback_out;
2959 }
2960 }
2961 Ok(())
2962 }
2963
2964 async fn resume_unconfirmed_commands(
2988 &mut self,
2989 results: &mut [(usize, SeatState, AgentOutcome)],
2990 sent: &[SeatJob],
2991 prompts: &Prompts,
2992 run_id: &str,
2993 ) {
2994 for (wi, seat, out) in results.iter_mut() {
2995 let AgentOutcome::Ok(o) = &*out else {
2996 continue;
2997 };
2998 if !has_unconfirmed_command(&o.commands) {
2999 continue;
3000 }
3001 let Some(job) = sent.get(*wi) else { continue };
3002 if !has_context(&job.spec, seat, job.sessions) {
3003 self.state.event(
3004 "implement",
3005 format!(
3006 "{}: the reply named a command whose own CLI never confirmed the exit \
3007 status of, but there is no session left to resume",
3008 seat.key
3009 ),
3010 );
3011 continue;
3012 }
3013 self.state.event(
3014 "implement",
3015 format!(
3016 "{}: the reply named a command whose own CLI never confirmed the exit \
3017 status of; resuming the conversation",
3018 seat.key
3019 ),
3020 );
3021 let mut retry = job.clone();
3022 retry.seat = seat.clone();
3023 retry.prompt = prompt::resume_incomplete(
3024 "a command in your last reply had no confirmed exit status",
3025 );
3026 retry.timeout = retry_budget(job.timeout, true);
3027 retry.stem = format!("{}-confirm", job.stem);
3028 let cache = self.state.config.cache_dir();
3029 let ctx = WaveCtx {
3030 carry_seats: false,
3031 pauses: Some((&self.pause, &self.interrupt)),
3032 run: run_id,
3033 node: "implement",
3034 prompts,
3035 cache: cache.as_deref(),
3036 round: None,
3037 };
3038 let (resumed_seat, resumed) =
3039 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3040 *seat = resumed_seat;
3041 *out = resumed;
3042 }
3043 }
3044
3045 async fn continue_fix_report(
3066 &mut self,
3067 mut seat: SeatState,
3068 parse_err: String,
3069 job: &SeatJob,
3070 prompts: &Prompts,
3071 run_id: &str,
3072 round: usize,
3073 ) -> (
3074 SeatState,
3075 Option<FixReport>,
3076 Option<String>,
3077 ContinuationRecord,
3078 ) {
3079 let mut last_err = parse_err;
3080 let mut cumulative_wait_ms = 0u64;
3081 let mut attempts = 0usize;
3082 loop {
3083 if !has_context(&job.spec, &seat, job.sessions) {
3084 self.state.event(
3085 "fix",
3086 format!(
3087 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3088 session left to resume into"
3089 ),
3090 );
3091 let outcome = if attempts == 0 {
3092 ContinuationOutcome::NoSession
3093 } else {
3094 ContinuationOutcome::Exhausted
3095 };
3096 return (
3097 seat,
3098 None,
3099 Some(format!("unparsable fix report: {last_err}")),
3100 ContinuationRecord {
3101 attempts,
3102 cumulative_wait_ms,
3103 outcome,
3104 },
3105 );
3106 }
3107 if attempts >= MAX_FIX_CONTINUATIONS {
3108 self.state.event(
3109 "fix",
3110 format!(
3111 "round {round}: fixer's reply still had no adoption report after \
3112 {attempts} continuation(s) ({last_err}); giving up"
3113 ),
3114 );
3115 return (
3116 seat,
3117 None,
3118 Some(format!(
3119 "unparsable fix report after {attempts} continuation(s): {last_err}"
3120 )),
3121 ContinuationRecord {
3122 attempts,
3123 cumulative_wait_ms,
3124 outcome: ContinuationOutcome::Exhausted,
3125 },
3126 );
3127 }
3128 attempts += 1;
3129 self.state.event(
3130 "fix",
3131 format!(
3132 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3133 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3134 ),
3135 );
3136 let mut retry = job.clone();
3137 retry.seat = seat.clone();
3138 retry.prompt = prompt::resume_incomplete(&last_err);
3139 retry.timeout = retry_budget(job.timeout, true);
3140 retry.stem = format!("{}-continue{attempts}", job.stem);
3141 let cache = self.state.config.cache_dir();
3142 let ctx = WaveCtx {
3143 carry_seats: false,
3144 pauses: Some((&self.pause, &self.interrupt)),
3145 run: run_id,
3146 node: "fix",
3147 prompts,
3148 cache: cache.as_deref(),
3149 round: Some(round),
3150 };
3151 let (resumed_seat, resumed_out) = run_one(
3152 retry,
3153 Arc::clone(&self.sem),
3154 &ctx,
3155 &mut self.state,
3156 attempts,
3157 )
3158 .await;
3159 seat = resumed_seat;
3160 match resumed_out {
3161 AgentOutcome::Ok(o) => {
3162 cumulative_wait_ms += o.duration_ms;
3163 match verdict::extract_json::<FixReport>(&o.text) {
3164 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3165 self.state.event(
3166 "fix",
3167 format!(
3168 "round {round}: fixer's adoption report recovered after \
3169 {attempts} continuation(s)"
3170 ),
3171 );
3172 return (
3173 seat,
3174 Some(report),
3175 None,
3176 ContinuationRecord {
3177 attempts,
3178 cumulative_wait_ms,
3179 outcome: ContinuationOutcome::Resumed,
3180 },
3181 );
3182 }
3183 Ok(_) => {
3191 last_err = "the reply parsed, but it reported a command whose own CLI \
3192 never confirmed an exit status"
3193 .to_owned();
3194 }
3195 Err(e) => last_err = e.to_string(),
3196 }
3197 }
3198 AgentOutcome::Quota(o) => {
3199 cumulative_wait_ms += o.duration_ms;
3200 self.state.quota.push(QuotaLoss {
3201 seat: seat.key.clone(),
3202 node: "fix".to_owned(),
3203 at: Timestamp::now(),
3204 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3205 });
3206 self.state.event(
3207 "fix",
3208 format!(
3209 "round {round}: continuation rate limited (quota); not retrying now"
3210 ),
3211 );
3212 return (
3213 seat,
3214 None,
3215 Some("rate limited (quota) while recovering the fix report".to_owned()),
3216 ContinuationRecord {
3217 attempts,
3218 cumulative_wait_ms,
3219 outcome: ContinuationOutcome::QuotaLost,
3220 },
3221 );
3222 }
3223 AgentOutcome::Dropped(o) => {
3224 cumulative_wait_ms += o.duration_ms;
3225 let why = o
3226 .dropped
3227 .as_ref()
3228 .map(|d| d.why.as_str())
3229 .unwrap_or("the CLI ended the stream without delivering its answer");
3230 last_err = format!("the CLI dropped the stream ({why})");
3231 }
3232 AgentOutcome::Failed(e) => last_err = e,
3233 AgentOutcome::ResourceBlocked(e) => {
3236 return (
3237 seat,
3238 None,
3239 Some(format!("the continuation was not started: {e}")),
3240 ContinuationRecord {
3241 attempts: attempts.saturating_sub(1),
3242 cumulative_wait_ms,
3243 outcome: ContinuationOutcome::NoSession,
3244 },
3245 );
3246 }
3247 }
3248 }
3249 }
3250
3251 fn after_implement(&mut self) -> Result<()> {
3252 if !self.state.candidate_preservation.is_empty() {
3253 bail!("candidate preservation incomplete; judging is unsafe");
3254 }
3255 if self.state.leaks.is_empty() {
3257 let cfg = self.state.config.blind.clone();
3258 let mut leaks = Vec::new();
3259 for c in &self.state.candidates {
3260 let Some(patch) =
3261 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3262 else {
3263 continue;
3264 };
3265 leaks.extend(blind::scan(
3266 &format!("candidate {} patch", c.label),
3267 &patch,
3268 &cfg.vendor_tokens,
3269 ));
3270 }
3271 if !leaks.is_empty() {
3272 let summary = leaks
3273 .iter()
3274 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3275 .collect::<Vec<_>>()
3276 .join(", ");
3277 match cfg.on_leak {
3278 LeakPolicy::Fail => {
3279 self.state.status = RunStatus::Failed;
3280 self.state
3281 .event("blind", format!("vendor text in a patch: {summary}"));
3282 self.state.leaks = leaks;
3283 self.state.save()?;
3284 self.settle_questions();
3285 bail!(
3286 "blind.on_leak = \"fail\" and vendor text reached a \
3287 judged patch: {summary}"
3288 );
3289 }
3290 LeakPolicy::Redact => self.state.event(
3291 "blind",
3292 format!("redacting vendor text for judging: {summary}"),
3293 ),
3294 LeakPolicy::Warn => self.state.event(
3295 "blind",
3296 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3297 ),
3298 }
3299 self.state.leaks = leaks;
3300 }
3301 }
3302
3303 if self.state.viable().is_empty() {
3304 if self.state.all_candidates_verified_noop() {
3305 self.state.status = RunStatus::VerifiedNoop;
3316 self.state.save()?;
3317 self.settle_questions();
3318 return Ok(());
3319 }
3320 self.state.status = RunStatus::Failed;
3321 self.state.save()?;
3322 self.settle_questions();
3323 bail!("no candidate produced a change; nothing to judge");
3324 }
3325 self.state.status = RunStatus::Judging;
3326 self.state.save()?;
3327 Ok(())
3328 }
3329
3330 async fn judge(&mut self) -> Result<()> {
3333 let run_id = self.state.id.clone();
3338 let prompts = self.state.config.prompts.clone();
3339 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3340 return Ok(());
3341 }
3342 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3343 if viable.len() == 1 {
3344 self.state.judge_skipped = true;
3351 self.state.event(
3352 "judge",
3353 format!(
3354 "only candidate {} produced a change; judging skipped",
3355 viable[0].label
3356 ),
3357 );
3358 self.state.save()?;
3359 return Ok(());
3360 }
3361 self.state.status = RunStatus::Judging;
3362
3363 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3364 let language = self.state.config.graph.language.clone();
3365 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3366 let sessions = self.state.config.graph.sessions;
3367 let artifacts = agent::artifacts_dir(&self.state.dir());
3368 let root = self.state.worktree_root();
3369 let base_short = short(&self.state.base_commit);
3370
3371 let mut jobs = Vec::new();
3372 let mut orders = Vec::new();
3373 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3374 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3375 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3376 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3377 let seat_key = format!("judge-{}", j + 1);
3378 let seat = self.seat(&seat_key, &spec.id);
3379 jobs.push(SeatJob {
3380 prompt: prompt::judge(
3381 &self.state.instruction,
3382 &views,
3383 self.roles.judges.len(),
3384 &base_short,
3385 &language,
3386 ),
3387 spec,
3388 seat,
3389 cwd: root.join(format!("judge-{}", j + 1)),
3390 timeout,
3391 allow_write: false,
3392 sessions,
3393 artifacts: artifacts.clone(),
3394 stem: format!("judge-{}", j + 1),
3395 handover: None,
3396 });
3397 }
3398
3399 self.state.event(
3400 "judge",
3401 format!(
3402 "{} judges ranking {} candidates blind",
3403 jobs.len(),
3404 viable.len()
3405 ),
3406 );
3407 let labels_for_check = labels.clone();
3408 let mut quota_losses = Vec::new();
3409 let cache = self.state.config.cache_dir();
3410 let ctx = WaveCtx {
3411 carry_seats: false,
3412 pauses: Some((&self.pause, &self.interrupt)),
3413 run: &run_id,
3414 node: "judge",
3415 prompts: &prompts,
3416 cache: cache.as_deref(),
3417 round: None,
3418 };
3419 let results = ask_json_wave::<Ranking>(
3420 jobs,
3421 Arc::clone(&self.sem),
3422 self.state.config.graph.retries,
3423 &self.roles.judge_roster,
3424 &ctx,
3425 &mut quota_losses,
3426 &mut self.state,
3427 &move |r: &Ranking| r.validate(&labels_for_check),
3428 )
3429 .await;
3430 if self.state.resource_block.is_some() {
3431 return self.stop_for_resource();
3432 }
3433 self.state.quota.extend(quota_losses);
3434
3435 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3436 let agent_id = seat.agent.clone();
3437 self.state.seats.insert(seat.key.clone(), seat);
3438 let mut record = Judgement {
3439 judge: j + 1,
3440 seat: format!("judge-{}", j + 1),
3441 agent: agent_id,
3442 ranking: Vec::new(),
3443 reasons: BTreeMap::new(),
3444 confidence: None,
3445 strengths: BTreeMap::new(),
3446 order: orders[j].clone(),
3447 failed: None,
3448 duration_ms: 0,
3449 };
3450 match res {
3451 Ok((ranking, out)) => {
3452 record.ranking = ranking.normalized();
3453 record.reasons = ranking.reasons;
3454 record.strengths = ranking.strengths;
3455 record.confidence = ranking.confidence;
3456 record.duration_ms = out.duration_ms;
3457 self.state.event(
3458 "judge",
3459 format!(
3460 "judge {} ranked {}",
3461 j + 1,
3462 record.ranking.iter().collect::<String>()
3463 ),
3464 );
3465 }
3466 Err(e) => {
3467 record.failed = Some(e.to_string());
3468 self.state
3469 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3470 }
3471 }
3472 self.state.judgements.push(record);
3473 self.state.save()?;
3474 }
3475 Ok(())
3476 }
3477
3478 async fn deliberate(&mut self) -> Result<()> {
3481 let run_id = self.state.id.clone();
3486 let prompts = self.state.config.prompts.clone();
3487 if !self.state.deliberation.is_empty() {
3488 return Ok(());
3489 }
3490 let tops: Vec<char> = self
3491 .state
3492 .judgements
3493 .iter()
3494 .filter_map(|j| j.ranking.first().copied())
3495 .collect();
3496 let rounds = self.state.config.graph.deliberate_rounds;
3497 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3498 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3499 self.state.event(
3500 "deliberate",
3501 format!("judges agreed on {} outright; no deliberation", tops[0]),
3502 );
3503 }
3504 self.state.status = RunStatus::Voting;
3505 self.state.save()?;
3506 return Ok(());
3507 }
3508
3509 self.state.status = RunStatus::Deliberating;
3510 self.state.event(
3511 "deliberate",
3512 format!(
3513 "split: first choices were {} — opening {rounds} round(s)",
3514 tops.iter().collect::<String>()
3515 ),
3516 );
3517
3518 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3519 let language = self.state.config.graph.language.clone();
3520 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3521 let sessions = self.state.config.graph.sessions;
3522 let artifacts = agent::artifacts_dir(&self.state.dir());
3523 let root = self.state.worktree_root();
3524 let base_short = short(&self.state.base_commit);
3525
3526 for round in 1..=rounds {
3530 let mut turns: Vec<DeliberationTurn> = Vec::new();
3531 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3532 if self.state.judgements[j].failed.is_some() {
3533 continue;
3534 }
3535 let seat_key = format!("judge-{}", j + 1);
3536 let spec = self.occupant(&seat_key, spec);
3537 let mut seat = self.seat(&seat_key, &spec.id);
3538 let transcript = self.transcript(&turns, j);
3539 let build = |context: Option<&str>| {
3540 prompt::deliberate(
3541 &self.state.instruction,
3542 context,
3543 &transcript,
3544 round,
3545 rounds,
3546 &language,
3547 )
3548 };
3549 let block = self.candidate_block(&viable, &base_short);
3550 let full = build(Some(&block));
3551 let text = if has_context(&spec, &seat, sessions) {
3552 build(None)
3553 } else {
3554 full.clone()
3555 };
3556 let job = SeatJob {
3557 spec,
3558 seat: seat.clone(),
3559 prompt: text,
3560 cwd: root.join(format!("judge-{}", j + 1)),
3561 timeout,
3562 allow_write: false,
3563 sessions,
3564 artifacts: artifacts.clone(),
3565 stem: format!("delib-{round}-judge-{}", j + 1),
3566 handover: Some(full),
3567 };
3568 let cache = self.state.config.cache_dir();
3569 let ctx = WaveCtx {
3570 carry_seats: false,
3571 pauses: Some((&self.pause, &self.interrupt)),
3572 run: &run_id,
3573 node: "deliberate",
3574 prompts: &prompts,
3575 cache: cache.as_deref(),
3576 round: None,
3577 };
3578 let mut losses = Vec::new();
3582 let mut results = ask_wave_with::<String>(
3583 vec![job],
3584 Arc::clone(&self.sem),
3585 0,
3586 &self.roles.judge_roster,
3587 &ctx,
3588 &mut losses,
3589 &mut self.state,
3590 &|text: &str| {
3591 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3592 },
3593 )
3594 .await;
3595 self.state.quota.extend(losses);
3596 let (updated, res, _) = results.pop().expect("one job in, one result out");
3597 seat = updated;
3598 let agent_id = seat.agent.clone();
3599 self.state.seats.insert(seat.key.clone(), seat);
3600 let body = match res {
3601 Ok((body, _)) => body,
3602 Err(e) => {
3605 self.state
3606 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3607 continue;
3608 }
3609 };
3610 let tentative = verdict::extract_json::<Position>(&body)
3611 .ok()
3612 .and_then(|p| p.tentative)
3613 .and_then(|s| s.trim().chars().next())
3614 .map(|c| c.to_ascii_uppercase());
3615 self.state.event(
3616 "deliberate",
3617 format!(
3618 "round {round}: judge {} now favours {}",
3619 j + 1,
3620 tentative.map_or("—".to_owned(), |c| c.to_string())
3621 ),
3622 );
3623 turns.push(DeliberationTurn {
3624 judge: j + 1,
3625 agent: agent_id,
3626 body: blind::sanitize_prose(&body, &self.state.config.blind),
3627 tentative,
3628 });
3629 }
3630 self.state
3631 .deliberation
3632 .push(DeliberationRound { round, turns });
3633 self.state.save()?;
3634 }
3635
3636 self.state.status = RunStatus::Voting;
3637 self.state.save()?;
3638 Ok(())
3639 }
3640
3641 async fn vote(&mut self) -> Result<()> {
3644 let run_id = self.state.id.clone();
3649 let prompts = self.state.config.prompts.clone();
3650 if !self.state.votes.is_empty() {
3651 return Ok(());
3652 }
3653 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3654 if viable.len() == 1 {
3655 return Ok(());
3656 }
3657 self.state.status = RunStatus::Voting;
3658
3659 let language = self.state.config.graph.language.clone();
3660 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3661 let sessions = self.state.config.graph.sessions;
3662 let artifacts = agent::artifacts_dir(&self.state.dir());
3663 let root = self.state.worktree_root();
3664 let base_short = short(&self.state.base_commit);
3665 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3666
3667 let mut jobs = Vec::new();
3668 let mut seats_at = Vec::new();
3669 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3670 if self
3671 .state
3672 .judgements
3673 .get(j)
3674 .is_some_and(|r| r.failed.is_some())
3675 {
3676 continue;
3677 }
3678 let seat_key = format!("judge-{}", j + 1);
3679 let spec = self.occupant(&seat_key, spec);
3680 let seat = self.seat(&seat_key, &spec.id);
3681 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3682 let text = if has_context(&spec, &seat, sessions) {
3683 prompt::final_vote(&viable, &language)
3684 } else {
3685 full.clone()
3686 };
3687 jobs.push(SeatJob {
3688 spec,
3689 seat,
3690 prompt: text,
3691 cwd: root.join(format!("judge-{}", j + 1)),
3692 timeout,
3693 allow_write: false,
3694 sessions,
3695 artifacts: artifacts.clone(),
3696 stem: format!("vote-judge-{}", j + 1),
3697 handover: Some(full),
3698 });
3699 seats_at.push(j);
3700 }
3701
3702 self.state.event(
3703 "vote",
3704 format!(
3705 "collecting {} final votes one by one, privately",
3706 jobs.len()
3707 ),
3708 );
3709 let allowed = viable.clone();
3710 let mut quota_losses = Vec::new();
3711 let cache = self.state.config.cache_dir();
3712 let ctx = WaveCtx {
3713 carry_seats: false,
3714 pauses: Some((&self.pause, &self.interrupt)),
3715 run: &run_id,
3716 node: "vote",
3717 prompts: &prompts,
3718 cache: cache.as_deref(),
3719 round: None,
3720 };
3721 let results = ask_json_wave::<FinalVote>(
3722 jobs,
3723 Arc::clone(&self.sem),
3724 self.state.config.graph.retries,
3725 &self.roles.judge_roster,
3726 &ctx,
3727 &mut quota_losses,
3728 &mut self.state,
3729 &move |v: &FinalVote| match v.label() {
3730 Some(c) if allowed.contains(&c) => Ok(()),
3731 other => bail!("vote {other:?} is not one of {allowed:?}"),
3732 },
3733 )
3734 .await;
3735 if self.state.resource_block.is_some() {
3736 return self.stop_for_resource();
3737 }
3738 self.state.quota.extend(quota_losses);
3739
3740 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3741 let agent_id = seat.agent.clone();
3742 self.state.seats.insert(seat.key.clone(), seat);
3743 let initial = self
3744 .state
3745 .judgements
3746 .get(j)
3747 .and_then(|r| r.ranking.first().copied());
3748 let mut record = VoteRecord {
3749 judge: j + 1,
3750 agent: agent_id,
3751 vote: None,
3752 reason: String::new(),
3753 changed: false,
3754 };
3755 match res {
3756 Ok((v, _)) => {
3757 record.vote = v.label();
3758 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3759 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3760 self.state.event(
3761 "vote",
3762 format!(
3763 "judge {} voted {}{}",
3764 j + 1,
3765 record.vote.unwrap_or('?'),
3766 if record.changed { " (changed)" } else { "" }
3767 ),
3768 );
3769 }
3770 Err(e) => {
3771 self.state
3772 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3773 }
3774 }
3775 self.state.votes.push(record);
3776 self.state.save()?;
3777 }
3778 Ok(())
3779 }
3780
3781 fn tally(&mut self) -> Result<()> {
3784 if self.state.tally.is_some() {
3785 return Ok(());
3786 }
3787 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3788 let tops: Vec<char> = self
3789 .state
3790 .judgements
3791 .iter()
3792 .filter_map(|j| j.ranking.first().copied())
3793 .collect();
3794 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3795
3796 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3799 let mut cast: Vec<char> = Vec::new();
3800 for (i, j) in self.state.judgements.iter().enumerate() {
3801 let vote = self
3802 .state
3803 .votes
3804 .iter()
3805 .find(|v| v.judge == i + 1)
3806 .and_then(|v| v.vote)
3807 .or_else(|| j.ranking.first().copied());
3808 if let Some(v) = vote {
3809 *first_choice.entry(v).or_insert(0) += 1;
3810 cast.push(v);
3811 }
3812 }
3813
3814 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3815 for j in &self.state.judgements {
3816 let n = j.ranking.len();
3817 for (pos, label) in j.ranking.iter().enumerate() {
3818 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3819 }
3820 }
3821
3822 let best = first_choice.values().copied().max().unwrap_or(0);
3823 let mut leaders: Vec<char> = first_choice
3824 .iter()
3825 .filter(|(_, v)| **v == best)
3826 .map(|(k, _)| *k)
3827 .collect();
3828 let mut tie_break = None;
3829 if leaders.len() > 1 {
3830 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3831 let borda_leaders: Vec<char> = leaders
3832 .iter()
3833 .copied()
3834 .filter(|l| borda[l] == top_borda)
3835 .collect();
3836 tie_break = Some(if borda_leaders.len() == 1 {
3837 format!(
3838 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3839 leaders.len()
3840 )
3841 } else {
3842 format!(
3843 "{} way tie on both first-choice votes and Borda points, broken by label order",
3844 leaders.len()
3845 )
3846 });
3847 leaders = borda_leaders;
3848 leaders.sort_unstable();
3849 }
3850 let winner = *leaders
3851 .first()
3852 .or(viable.first())
3853 .context("no candidate to declare a winner from")?;
3854
3855 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3856 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3857 let deliberated = !self.state.deliberation.is_empty();
3858
3859 let quota_seats: std::collections::BTreeSet<&str> =
3863 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3864 let mut present = 0usize;
3865 for (i, j) in self.state.judgements.iter().enumerate() {
3866 if quota_seats.contains(j.seat.as_str()) {
3867 continue;
3868 }
3869 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3870 let voted = self
3871 .state
3872 .votes
3873 .iter()
3874 .any(|v| v.judge == i + 1 && v.vote.is_some());
3875 if ranked || voted {
3876 present += 1;
3877 }
3878 }
3879 let needs_quorum = viable.len() > 1;
3885 let judges_total = if needs_quorum {
3886 self.roles.judges.len()
3887 } else {
3888 0
3889 };
3890 let quorum = if needs_quorum {
3891 judges_total / 2 + 1
3892 } else {
3893 0
3894 };
3895 let met_quorum = !needs_quorum || present >= quorum;
3896 let uncontested = (!needs_quorum).then(|| {
3897 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3898 });
3899
3900 self.state.event(
3901 "tally",
3902 match &uncontested {
3903 Some(reason) => format!("winner {winner} — {reason}"),
3904 None => format!(
3905 "winner {winner} — votes {} | initial {} | {} changed | \
3906 {present}/{judges_total} judges{}",
3907 first_choice
3908 .iter()
3909 .map(|(k, v)| format!("{k}:{v}"))
3910 .collect::<Vec<_>>()
3911 .join(" "),
3912 if unanimous_initial {
3913 "unanimous"
3914 } else {
3915 "split"
3916 },
3917 changed_votes,
3918 if met_quorum {
3919 String::new()
3920 } else {
3921 format!(" — below quorum ({quorum} required)")
3922 },
3923 ),
3924 },
3925 );
3926 if !met_quorum {
3927 self.state.event(
3928 "stall",
3929 format!(
3930 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3931 the run stops here, resumable"
3932 ),
3933 );
3934 }
3935 self.state.tally = Some(Tally {
3936 first_choice,
3937 borda,
3938 winner,
3939 rankings: tops.len(),
3940 unanimous_initial,
3941 deliberated,
3942 changed_votes,
3943 unanimous_final,
3944 tie_break,
3945 judges: judges_total,
3946 present,
3947 quorum,
3948 met_quorum,
3949 uncontested,
3950 });
3951 self.state.status = if met_quorum {
3952 RunStatus::Reviewing
3953 } else {
3954 RunStatus::Stalled
3955 };
3956 self.state.save()?;
3957 Ok(())
3958 }
3959
3960 #[allow(clippy::too_many_lines)]
3981 async fn recover_stall(&mut self) -> Result<bool> {
3982 let run_id = self.state.id.clone();
3987 let prompts = self.state.config.prompts.clone();
3988 let quota_seats: BTreeSet<&str> =
3993 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3994 let absent: Vec<String> = self
3995 .state
3996 .judgements
3997 .iter()
3998 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3999 .map(|j| j.seat.clone())
4000 .collect();
4001 if absent.is_empty() {
4002 return Ok(false);
4003 }
4004 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4005 if viable.len() <= 1 {
4006 return Ok(false);
4007 }
4008 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4009 let language = self.state.config.graph.language.clone();
4010 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4011 let sessions = self.state.config.graph.sessions;
4012 let artifacts = agent::artifacts_dir(&self.state.dir());
4013 let root = self.state.worktree_root();
4014 let base_short = short(&self.state.base_commit);
4015 let candidates: Vec<Candidate> = viable.clone();
4016
4017 let mut positions: Vec<usize> = absent
4019 .iter()
4020 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4021 .collect();
4022 if positions.is_empty() {
4023 return Ok(false);
4024 }
4025 positions.sort_unstable();
4026 positions.dedup();
4027
4028 let mut judge_jobs = Vec::new();
4030 for &j in &positions {
4031 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4032 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4033 let seat_key = format!("judge-{}", j + 1);
4034 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4035 let seat = self.seat(&seat_key, &spec.id);
4036 judge_jobs.push(SeatJob {
4037 spec,
4038 seat,
4039 prompt: prompt::judge(
4040 &self.state.instruction,
4041 &views,
4042 self.roles.judges.len(),
4043 &base_short,
4044 &language,
4045 ),
4046 cwd: root.join(seat_key),
4047 timeout,
4048 allow_write: false,
4049 sessions,
4050 artifacts: artifacts.clone(),
4051 stem: format!("judge-{}-recover", j + 1),
4052 handover: None,
4053 });
4054 }
4055
4056 let labels_for_check = labels.clone();
4057 let mut judge_losses = Vec::new();
4058 let retries = self.state.config.graph.retries;
4059 let cache = self.state.config.cache_dir();
4060 let ctx = WaveCtx {
4061 carry_seats: false,
4062 pauses: Some((&self.pause, &self.interrupt)),
4063 run: &run_id,
4064 node: "judge",
4065 prompts: &prompts,
4066 cache: cache.as_deref(),
4067 round: None,
4068 };
4069 let results = ask_json_wave::<Ranking>(
4070 judge_jobs,
4071 Arc::clone(&self.sem),
4072 retries,
4073 &self.roles.judge_roster,
4074 &ctx,
4075 &mut judge_losses,
4076 &mut self.state,
4077 &move |r: &Ranking| r.validate(&labels_for_check),
4078 )
4079 .await;
4080 if self.state.resource_block.is_some() {
4081 self.stop_for_resource()?;
4082 return Ok(false);
4083 }
4084
4085 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4087 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4088 let agent_id = seat.agent.clone();
4089 self.state.seats.insert(seat.key.clone(), seat);
4090 let record = &mut self.state.judgements[j];
4091 match res {
4092 Ok((ranking, out)) => {
4093 record.agent = agent_id;
4094 record.ranking = ranking.normalized();
4095 record.reasons = ranking.reasons;
4096 record.strengths = ranking.strengths;
4097 record.confidence = ranking.confidence;
4098 record.failed = None;
4099 record.duration_ms = out.duration_ms;
4100 recovered.insert(j);
4101 self.state.event(
4102 "recover",
4103 format!("judge {} ranked again after the limit", j + 1),
4104 );
4105 }
4106 Err(e) => {
4107 self.state
4108 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4109 }
4110 }
4111 }
4112
4113 let mut vote_jobs = Vec::new();
4115 let mut vote_pos: Vec<usize> = Vec::new();
4116 for &j in &recovered {
4117 let seat_key = format!("judge-{}", j + 1);
4118 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4119 let seat = self.seat(&seat_key, &spec.id);
4120 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4121 let text = if has_context(&spec, &seat, sessions) {
4122 prompt::final_vote(&labels, &language)
4123 } else {
4124 full.clone()
4125 };
4126 vote_jobs.push(SeatJob {
4127 spec,
4128 seat,
4129 prompt: text,
4130 cwd: root.join(seat_key),
4131 timeout,
4132 allow_write: false,
4133 sessions,
4134 artifacts: artifacts.clone(),
4135 stem: format!("vote-judge-{}-recover", j + 1),
4136 handover: Some(full),
4137 });
4138 vote_pos.push(j);
4139 }
4140 let allowed = labels.clone();
4141 let mut vote_losses = Vec::new();
4142 let vote_retries = self.state.config.graph.retries;
4143 let vote_cache = self.state.config.cache_dir();
4144 let ctx = WaveCtx {
4145 carry_seats: false,
4146 pauses: Some((&self.pause, &self.interrupt)),
4147 run: &run_id,
4148 node: "vote",
4149 prompts: &prompts,
4150 cache: vote_cache.as_deref(),
4151 round: None,
4152 };
4153 let votes = ask_json_wave::<FinalVote>(
4154 vote_jobs,
4155 Arc::clone(&self.sem),
4156 vote_retries,
4157 &self.roles.judge_roster,
4158 &ctx,
4159 &mut vote_losses,
4160 &mut self.state,
4161 &move |v: &FinalVote| match v.label() {
4162 Some(c) if allowed.contains(&c) => Ok(()),
4163 other => bail!("vote {other:?} is not one of {allowed:?}"),
4164 },
4165 )
4166 .await;
4167 if self.state.resource_block.is_some() {
4168 self.stop_for_resource()?;
4169 return Ok(false);
4170 }
4171 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4172 let agent_id = seat.agent.clone();
4173 self.state.seats.insert(seat.key.clone(), seat);
4174 match res {
4175 Ok((v, _)) => {
4176 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4177 rec.vote = v.label();
4178 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4179 } else {
4180 self.state.votes.push(VoteRecord {
4181 judge: j + 1,
4182 agent: agent_id,
4183 vote: v.label(),
4184 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4185 changed: false,
4186 });
4187 }
4188 self.state.event(
4189 "recover",
4190 format!("judge {} voted again after the limit", j + 1),
4191 );
4192 }
4193 Err(e) => {
4194 self.state
4195 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4196 }
4197 }
4198 }
4199
4200 let recovered_keys: BTreeSet<String> = recovered
4204 .iter()
4205 .map(|&j| format!("judge-{}", j + 1))
4206 .collect();
4207 self.state
4208 .quota
4209 .retain(|q| !recovered_keys.contains(&q.seat));
4210 for loss in judge_losses.into_iter().chain(vote_losses) {
4214 if recovered_keys.contains(&loss.seat) {
4215 continue;
4216 }
4217 self.state.quota.retain(|q| q.seat != loss.seat);
4218 self.state.quota.push(loss);
4219 }
4220
4221 self.state.tally = None;
4223 self.tally()?;
4224 Ok(self
4225 .state
4226 .tally
4227 .as_ref()
4228 .map(|t| t.met_quorum)
4229 .unwrap_or(false))
4230 }
4231
4232 async fn graft(&mut self) -> Result<()> {
4248 let Err(e) = self.graft_inner().await else {
4249 return Ok(());
4250 };
4251 if e.is::<crate::repo_lock::ResourceBlocked>() {
4252 return Err(e);
4253 }
4254 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4255 self.state.status = RunStatus::Blocked;
4256 self.state.event("graft", format!("blocked: {e:#}"));
4257 self.state.save()?;
4258 return Err(e);
4259 }
4260 if let Some(rec) = self.state.graft.clone()
4265 && !rec.outcome.settled()
4266 {
4267 if let Some(winner) = self.state.winner().cloned() {
4268 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4269 if re.is::<crate::repo_lock::ResourceBlocked>() {
4270 return Err(re);
4271 }
4272 self.state.status = RunStatus::Blocked;
4273 self.state.event("graft", format!("blocked: {re:#}"));
4274 self.state.save()?;
4275 return Err(re);
4276 }
4277 }
4278 if let Some(g) = self.state.graft.as_mut() {
4279 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4280 }
4281 }
4282 self.state.event("graft", format!("skipped: {e:#}"));
4283 if let Err(e) = self.state.save() {
4284 tracing::warn!("save after graft: {e:#}");
4285 }
4286 Ok(())
4287 }
4288
4289 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4292 let _coordination = self.coordinate(wt).await?;
4293 crate::recovery::restore_repair(wt, pre)
4294 .await
4295 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4296 }
4297
4298 fn graft_skipped(&mut self, why: &str) {
4300 let message = format!("skipped: {why}");
4301 if !self
4302 .state
4303 .events
4304 .iter()
4305 .any(|e| e.node == "graft" && e.message == message)
4306 {
4307 self.state.event("graft", message);
4308 }
4309 }
4310
4311 async fn graft_inner(&mut self) -> Result<()> {
4312 if let Some(rec) = self.state.graft.clone()
4315 && !rec.outcome.settled()
4316 {
4317 if self.state.reviews.is_empty()
4321 && let Some(winner) = self.state.winner().cloned()
4322 {
4323 self.graft_restore(&winner.worktree, &rec.pre).await?;
4324 }
4325 if rec.outcome == GraftOutcome::WaitingForRestore {
4326 self.state.graft = None;
4327 self.state.save()?;
4328 } else {
4329 if let Some(g) = self.state.graft.as_mut() {
4330 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4331 }
4332 self.state.event(
4333 "graft",
4334 "rolled back: the previous round was interrupted; the winner is untouched",
4335 );
4336 self.state.save()?;
4337 return Ok(());
4338 }
4339 }
4340 let plan = match graft::plan(&self.state) {
4341 Ok(plan) => plan,
4342 Err(skip) => {
4343 if skip != graft::Skip::AlreadyDone {
4344 self.graft_skipped(skip.reason());
4345 self.state.save()?;
4346 }
4347 return Ok(());
4348 }
4349 };
4350 let Some(winner) = self.state.winner().cloned() else {
4351 return Ok(());
4352 };
4353 let wt = winner.worktree.clone();
4354 if !git::is_clean(&wt).await? {
4355 self.graft_skipped("the winner's worktree has uncommitted changes");
4356 self.state.save()?;
4357 return Ok(());
4358 }
4359 let pre = git::rev_parse(&wt, "HEAD").await?;
4360 self.state.graft = Some(GraftRecord {
4363 pre: pre.clone(),
4364 post: None,
4365 lines: None,
4366 limit: None,
4367 agent: None,
4368 outcome: GraftOutcome::Running,
4369 requested: graft::requested(&plan),
4370 adoption: None,
4371 adoption_error: None,
4372 });
4373 self.state.event(
4374 "graft",
4375 format!(
4376 "porting strengths of candidate(s) {} onto {}",
4377 plan.strengths
4378 .iter()
4379 .map(|(l, _, _)| l.to_string())
4380 .collect::<Vec<_>>()
4381 .join(", "),
4382 plan.winner
4383 ),
4384 );
4385 self.state.save()?;
4386
4387 let fix_prompt = prompt::graft(
4388 &self.state.instruction,
4389 plan.winner,
4390 &plan.strengths,
4391 &self.state.config.graph.language,
4392 );
4393 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4394 let sessions = self.state.config.graph.sessions;
4395 let artifacts = agent::artifacts_dir(&self.state.dir());
4396 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4397 let (_, seat, out) = self
4398 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4399 SeatJob {
4400 prompt: fix_prompt.clone(),
4401 spec,
4402 seat,
4403 cwd: wt.clone(),
4404 timeout,
4405 allow_write: true,
4406 sessions,
4407 artifacts: artifacts.clone(),
4408 stem: "graft".to_owned(),
4409 handover: None,
4410 }
4411 })
4412 .await;
4413 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4414 self.state.graft.as_mut().expect("graft record").outcome =
4417 GraftOutcome::WaitingForRestore;
4418 return self.stop_for_resource();
4419 }
4420 let mut why: Option<String> = None;
4421 match &out {
4422 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4423 why = Some("the fixer produced nothing usable".to_owned());
4424 }
4425 AgentOutcome::Ok(_) => {}
4426 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4427 AgentOutcome::Quota(o) => {
4428 self.state.quota.push(QuotaLoss {
4429 seat: seat.key.clone(),
4430 node: "graft".to_owned(),
4431 at: Timestamp::now(),
4432 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4433 });
4434 why = Some("rate limited (quota)".to_owned());
4435 }
4436 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4437 why = Some(format!(
4438 "the fixer failed: {}",
4439 e.lines()
4440 .next()
4441 .unwrap_or("")
4442 .chars()
4443 .take(160)
4444 .collect::<String>()
4445 ));
4446 }
4447 }
4448 let agent_id = seat.agent.clone();
4449 self.state.seats.insert(seat.key.clone(), seat);
4450
4451 let mut post = None;
4452 let mut lines = None;
4453 let _coordination = self.coordinate(&wt).await?;
4454 if why.is_none() {
4455 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4456 if let Ok(r) = &rescued {
4457 self.state.note_withheld("graft", &r.withheld);
4458 }
4459 let head = git::rev_parse(&wt, "HEAD").await?;
4460 let dirty = match &rescued {
4464 Ok(r) => git::status_porcelain(&wt)
4465 .await
4466 .map(|st| {
4467 st.lines().any(|l| {
4468 !r.withheld
4469 .iter()
4470 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4471 })
4472 })
4473 .unwrap_or(true),
4474 Err(_) => true,
4475 };
4476 if dirty {
4477 why = Some("the fixer left changes that could not be committed".to_owned());
4478 } else if head == pre {
4479 why = Some("the fixer changed nothing".to_owned());
4480 } else {
4481 lines = git::changed_lines(&wt, &pre, &head).await;
4482 post = Some(head);
4483 }
4484 }
4485 let (adoption, adoption_error) = match &out {
4486 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4487 _ => (None, None),
4488 };
4489 let requested = graft::requested(&plan);
4490 let outcome = match why {
4491 None => {
4492 self.state.event(
4493 "graft",
4494 format!(
4495 "applied: {}",
4496 lines.map_or_else(
4497 || "changed lines unavailable".to_owned(),
4498 |n| format!("{n} changed lines")
4499 )
4500 ),
4501 );
4502 GraftOutcome::Applied
4503 }
4504 Some(why) => {
4505 crate::recovery::restore_repair(&wt, &pre)
4508 .await
4509 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4510 self.state.event(
4511 "graft",
4512 format!("rolled back: {why}; the winner is untouched"),
4513 );
4514 GraftOutcome::RolledBack(why)
4515 }
4516 };
4517 self.state.graft = Some(GraftRecord {
4518 pre,
4519 post,
4520 lines,
4521 limit: None,
4522 agent: Some(agent_id),
4523 outcome,
4524 requested,
4525 adoption,
4526 adoption_error,
4527 });
4528 self.state.save()?;
4529 Ok(())
4530 }
4531
4532 async fn fold_losers(&mut self) -> Result<()> {
4535 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4536 return Ok(());
4537 };
4538 if !self.state.candidate_preservation.is_empty() {
4539 bail!("candidate preservation incomplete; refusing cleanup");
4540 }
4541 let repo = self.state.repo.clone();
4542 let _coordination = self.coordinate(&repo).await?;
4543 let mut folded = Vec::new();
4544 for i in 0..self.state.candidates.len() {
4545 let c = &self.state.candidates[i];
4546 if c.label == winner || c.folded {
4547 continue;
4548 }
4549 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4550 remove_preserved_worktree(&repo, &wt).await?;
4551 if git::branch_exists(&repo, &branch).await?
4552 && !git::branch_delete(&repo, &branch).await?
4553 {
4554 bail!("candidate branch removal refused");
4555 }
4556 self.state.candidates[i].folded = true;
4557 self.state.save()?;
4558 folded.push(label.to_string());
4559 }
4560 let root = self.state.worktree_root();
4562 for j in 1..=self.roles.judges.len() {
4563 let wt = root.join(format!("judge-{j}"));
4564 if wt.exists() {
4565 remove_preserved_worktree(&repo, &wt).await?;
4566 }
4567 }
4568 if self.state.config.graph.advise {
4571 for k in 1..=self.state.config.graph.advisors {
4572 let wt = root.join(format!("advisor-{k}"));
4573 if wt.exists() {
4574 remove_preserved_worktree(&repo, &wt).await?;
4575 }
4576 }
4577 }
4578 if !folded.is_empty() {
4579 self.state
4580 .event("fold", format!("folded candidates {}", folded.join(", ")));
4581 self.state.save()?;
4582 }
4583 Ok(())
4584 }
4585
4586 async fn sync_to_base(&mut self) -> Result<()> {
4626 if self.state.status == RunStatus::AlreadyInBase {
4627 return Ok(());
4628 }
4629 let conflicted = self
4630 .state
4631 .base_sync
4632 .as_ref()
4633 .is_some_and(|s| s.conflict.is_some());
4634 let Some(winner) = self.state.winner().cloned() else {
4635 return Ok(());
4636 };
4637
4638 let repo = self.state.repo.clone();
4639 let remote = self.state.config.merge.remote.clone();
4640 let base_branch = self.state.base_branch.clone();
4641 let tracking = format!("{remote}/{base_branch}");
4642
4643 git::fetch(&repo, &remote, &base_branch).await.ok();
4644 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4649 return Ok(());
4650 };
4651
4652 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4653 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4654 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4655
4656 if (behind > 0 || conflicted || head == tip)
4665 && self
4666 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4667 .await?
4668 {
4669 return Ok(());
4670 }
4671 if conflicted {
4672 return Ok(());
4673 }
4674
4675 if behind == 0 {
4676 if let Some(from) = self
4685 .state
4686 .rebase_fixes
4687 .iter()
4688 .rev()
4689 .find_map(|r| r.from.clone())
4690 && from != head
4691 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4692 .await
4693 .is_ok_and(|o| o.ok())
4694 {
4695 git::sync_to_head(&winner.worktree).await?;
4696 }
4697 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4703 if let Some(why) = &conflict {
4704 self.state.status = RunStatus::Blocked;
4705 self.state.event("land", why.clone());
4706 }
4707 self.state.base_sync = Some(BaseSync {
4708 tip,
4709 behind: 0,
4710 attempts,
4711 conflict,
4712 already_in: None,
4713 });
4714 self.state.save()?;
4715 return Ok(());
4716 }
4717
4718 if attempts >= BASE_SYNC_ROUNDS {
4719 let why = format!(
4720 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4721 rebase(s); rebasing again would only race it",
4722 winner.branch
4723 );
4724 self.state.status = RunStatus::Blocked;
4725 self.state.base_sync = Some(BaseSync {
4726 tip,
4727 behind,
4728 attempts,
4729 conflict: Some(why.clone()),
4730 already_in: None,
4731 });
4732 self.state.event("land", why);
4733 self.state.save()?;
4734 return Ok(());
4735 }
4736
4737 self.state.event(
4738 "land",
4739 format!(
4740 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4741 winner.branch
4742 ),
4743 );
4744 self.state.save()?;
4745
4746 let branch_tracking = format!("{remote}/{}", winner.branch);
4752 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4753 let remote_tip = if self.state.task_pr_branch().is_none()
4754 && matches!(&fetched_branch, Ok(o) if o.ok())
4755 {
4756 git::rev_parse(&repo, &branch_tracking).await.ok()
4757 } else {
4758 None
4759 };
4760 if let Some(theirs) = &remote_tip
4764 && !git::is_ancestor(&repo, theirs, &head).await
4765 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4766 .await
4767 .is_ok_and(|missing| missing.is_empty())
4768 {
4769 let why = format!(
4770 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4771 them",
4772 short(theirs),
4773 winner.branch
4774 );
4775 self.state.status = RunStatus::Blocked;
4776 self.state.base_sync = Some(BaseSync {
4777 tip,
4778 behind,
4779 attempts,
4780 conflict: Some(why.clone()),
4781 already_in: None,
4782 });
4783 self.state.event("land", why);
4784 self.state.save()?;
4785 return Ok(());
4786 }
4787
4788 let scratch = self.state.dir().join("base-sync");
4789 let rebased = match crate::rebase::rebase_with_fixer(
4790 &mut self.state,
4791 &scratch,
4792 &winner.branch,
4793 &tracking,
4794 )
4795 .await
4796 {
4797 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4798 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4799 Err(e) => Err(e),
4800 };
4801 let attempts = attempts + 1;
4802 match rebased {
4803 Ok(None) => {
4804 git::sync_to_head(&winner.worktree).await?;
4808 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4809 let mut conflict = None;
4810 if let Some(pinned) = &remote_tip {
4811 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4812 match pushed {
4813 Ok(o) if o.ok() => self.state.event(
4814 "land",
4815 format!("pushed rebased {} to {remote}", winner.branch),
4816 ),
4817 Ok(o) => {
4818 conflict = Some(format!(
4819 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4820 winner.branch,
4821 short(pinned),
4822 o.stderr.chars().take(600).collect::<String>()
4823 ));
4824 }
4825 Err(e) => {
4826 conflict = Some(format!(
4827 "rebased {} locally but could not push it: {e:#}",
4828 winner.branch
4829 ));
4830 }
4831 }
4832 }
4833 if let Some(why) = &conflict {
4834 self.state.status = RunStatus::Blocked;
4835 self.state.event("land", why.clone());
4836 }
4837 self.state.base_sync = Some(BaseSync {
4838 tip: tip.clone(),
4839 behind: 0,
4840 attempts,
4841 conflict,
4842 already_in: None,
4843 });
4844 self.state
4845 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4846 }
4847 Ok(Some(conflict)) => {
4848 let why = format!(
4849 "{} conflicts with {tracking} and did not rebase: {}",
4850 winner.branch,
4851 conflict.chars().take(600).collect::<String>()
4852 );
4853 self.state.status = RunStatus::Blocked;
4854 self.state.base_sync = Some(BaseSync {
4855 tip,
4856 behind,
4857 attempts,
4858 conflict: Some(why.clone()),
4859 already_in: None,
4860 });
4861 self.state.event("land", why);
4862 }
4863 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4867 self.state.save()?;
4868 return Ok(());
4869 }
4870 Err(e) => {
4871 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4872 self.state.status = RunStatus::Blocked;
4873 self.state.base_sync = Some(BaseSync {
4874 tip,
4875 behind,
4876 attempts,
4877 conflict: Some(why.clone()),
4878 already_in: None,
4879 });
4880 self.state.event("land", why);
4881 }
4882 }
4883 self.state.save()?;
4884 Ok(())
4885 }
4886
4887 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4893 if self.state.task_pr_branch().is_some() {
4896 return None;
4897 }
4898 let repo = self.state.repo.clone();
4899 let remote = self.state.config.merge.remote.clone();
4900 let fetched = git::fetch(&repo, &remote, branch).await;
4901 if !matches!(&fetched, Ok(o) if o.ok()) {
4902 return None;
4903 }
4904 let branch_tracking = format!("{remote}/{branch}");
4905 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4906 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4907 return None;
4908 }
4909 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4910 .await
4911 .is_ok_and(|missing| missing.is_empty())
4912 {
4913 return Some(format!(
4914 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4915 them",
4916 short(&theirs)
4917 ));
4918 }
4919 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4920 Ok(o) if o.ok() => {
4921 self.state
4922 .event("land", format!("pushed rebased {branch} to {remote}"));
4923 None
4924 }
4925 Ok(o) => Some(format!(
4926 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4927 someone may have pushed): {}",
4928 short(&theirs),
4929 o.stderr.chars().take(600).collect::<String>()
4930 )),
4931 Err(e) => Some(format!(
4932 "{branch} is rebased locally but could not be pushed: {e:#}"
4933 )),
4934 }
4935 }
4936
4937 async fn settle_already_in(
4954 &mut self,
4955 branch: &str,
4956 tip: &str,
4957 head: &str,
4958 attempts: usize,
4959 behind: usize,
4960 ) -> Result<bool> {
4961 let repo = self.state.repo.clone();
4962 let remote = self.state.config.merge.remote.clone();
4963 let start = self.state.base_commit.clone();
4964 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4965 Ok(Some(e)) => e,
4966 Ok(None) => return Ok(false),
4967 Err(e) => {
4968 tracing::warn!("already-in-base check for {branch}: {e:#}");
4969 return Ok(false);
4970 }
4971 };
4972 let mut verified = vec![head.to_owned()];
4973 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4974 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4975 if matches!(&fetched, Ok(o) if o.ok())
4976 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4977 && theirs != head
4978 {
4979 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4980 Ok(Some(_)) => verified.push(theirs),
4981 _ => return Ok(false),
4982 }
4983 }
4984 let base_branch = self.state.base_branch.clone();
4985 let message = format!(
4986 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4987 land",
4988 evidence.names(),
4989 evidence.proof.as_str()
4990 );
4991 let closed =
4992 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4993 .await;
4994 match closed {
4995 Ok(Ok(url)) => self
4996 .state
4997 .event("land", format!("closed {url}: superseded on {base_branch}")),
4998 Ok(Err(why)) => self
4999 .state
5000 .event("land", format!("did not close a pull request: {why}")),
5001 Err(e) => {
5002 let why = format!(
5003 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5004 not be closed ({e:#}); resume to retry"
5005 );
5006 self.state.status = RunStatus::Blocked;
5007 self.state.base_sync = Some(BaseSync {
5008 tip: tip.to_owned(),
5009 behind,
5010 attempts,
5011 conflict: Some(why.clone()),
5012 already_in: None,
5013 });
5014 self.state.event("land", why);
5015 self.state.save()?;
5016 return Ok(true);
5017 }
5018 }
5019 self.state.status = RunStatus::AlreadyInBase;
5020 self.state.base_sync = Some(BaseSync {
5021 tip: tip.to_owned(),
5022 behind,
5023 attempts,
5024 conflict: None,
5025 already_in: Some(evidence),
5026 });
5027 self.state.event("land", message);
5028 self.state.save()?;
5029 self.settle_questions();
5030 Ok(true)
5031 }
5032
5033 fn landing_base(&self) -> String {
5043 self.state
5044 .base_sync
5045 .as_ref()
5046 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5047 }
5048
5049 pub async fn fix_selected(
5082 &mut self,
5083 ids: &[String],
5084 reason: &str,
5085 allow_stale: bool,
5086 ) -> Result<()> {
5087 let reason = reason.trim();
5088 if reason.is_empty() {
5089 bail!("a fix request needs a reason — that is the operator's own record of why");
5090 }
5091 if ids.is_empty() {
5092 bail!("no finding id given");
5093 }
5094 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5095 bail!(
5096 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5097 has already concluded — can be given a targeted fix. A run still \
5098 in progress should simply be resumed; a `merged` run's branch has \
5099 already landed, so its answer is a fresh `magi review <branch>`, \
5100 not reopening this run's own record",
5101 self.state.id,
5102 self.state.status.as_str()
5103 );
5104 }
5105 let Some(winner) = self.state.winner().cloned() else {
5106 bail!("run {} has no winning candidate to fix", self.state.id);
5107 };
5108 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5109 bail!(
5110 "branch `{}` no longer exists; this run cannot be extended",
5111 winner.branch
5112 );
5113 }
5114 let home = crate::run::home();
5115 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5116 bail!(
5117 "run {} is currently being worked on by another magi process",
5118 self.state.id
5119 );
5120 }
5121 let _claim = FixClaim::acquire(&self.state.dir())?;
5127
5128 let mut seen = BTreeSet::new();
5132 let mut findings = Vec::new();
5133 let mut missing = Vec::new();
5134 for id in ids {
5135 if !seen.insert(id.clone()) {
5136 continue;
5137 }
5138 match self.state.finding(id) {
5139 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5140 id: f.id.clone(),
5141 severity: f.severity,
5142 reviewer_vote: rec.vote,
5143 round: round.round,
5144 round_head: round.head.clone(),
5145 reviewer: rec.reviewer,
5146 agent: rec.agent.clone(),
5147 file: f.file.clone(),
5148 line: f.line,
5149 title: f.title.clone(),
5150 detail: f.detail.clone(),
5151 outcome: OperatorFixOutcome::Pending,
5152 }),
5153 None => missing.push(id.clone()),
5154 }
5155 }
5156 if !missing.is_empty() {
5157 bail!(
5158 "unknown finding id(s): {}; nothing was changed",
5159 missing.join(", ")
5160 );
5161 }
5162
5163 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5164 let stale_details: Vec<(String, String)> = findings
5165 .iter()
5166 .filter(|f| f.round_head != head_at_request)
5167 .map(|f| (f.id.clone(), f.round_head.clone()))
5168 .collect();
5169 let stale = !stale_details.is_empty();
5170 if stale && !allow_stale {
5171 bail!(
5172 "the branch has moved since some finding(s) were raised — {} — now \
5173 at {}; pass --allow-stale to fix anyway, or re-run review first",
5174 stale_details
5175 .iter()
5176 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5177 .collect::<Vec<_>>()
5178 .join(", "),
5179 short(&head_at_request)
5180 );
5181 }
5182
5183 let request = OperatorFixRequest {
5184 requested_at: Timestamp::now(),
5185 reason: reason.to_owned(),
5186 findings,
5187 head_at_request: head_at_request.clone(),
5188 allow_stale,
5189 stale,
5190 fix: None,
5191 result_head: None,
5192 follow_up_review_run: None,
5193 };
5194 self.state.event(
5195 "fix",
5196 format!(
5197 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5198 request.findings.len(),
5199 request
5200 .findings
5201 .iter()
5202 .map(|f| f.id.as_str())
5203 .collect::<Vec<_>>()
5204 .join(", "),
5205 ),
5206 );
5207 self.state.operator_fixes.push(request);
5214 self.state.save()?;
5215 let request_index = self.state.operator_fixes.len() - 1;
5216
5217 let repo = self.state.repo.clone();
5226 let coordination = self.coordinate(&repo).await?;
5227 if winner.worktree.exists() {
5228 let dirty = git::git(
5231 &winner.worktree,
5232 &["status", "--porcelain", "--untracked-files=all"],
5233 )
5234 .await?;
5235 let only_withheld = dirty.lines().all(|l| {
5236 l.strip_prefix("?? ")
5237 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5238 });
5239 if !only_withheld {
5240 bail!(
5241 "`{}` has uncommitted changes; refusing to touch it — commit or \
5242 discard them first",
5243 winner.worktree.display()
5244 );
5245 }
5246 git::worktree_remove(&self.state.repo, &winner.worktree)
5247 .await
5248 .ok();
5249 }
5250 let fix_worktree = self.state.worktree_root().join("operator-fix");
5251 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5252 git::git(
5253 &self.state.repo,
5254 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5255 )
5256 .await
5257 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5258 if !git::is_clean(&fix_worktree).await? {
5259 git::worktree_remove(&self.state.repo, &fix_worktree)
5260 .await
5261 .ok();
5262 bail!(
5263 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5264 winner.branch
5265 );
5266 }
5267 let repo_for_setup = self.state.repo.clone();
5268 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5269 Ok(()) => {
5270 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5271 }
5272 Err(e) => Err(e),
5273 };
5274 if let Err(e) = setup {
5275 git::worktree_remove(&self.state.repo, &fix_worktree)
5276 .await
5277 .ok();
5278 return Err(e);
5279 }
5280
5281 let run_id = self.state.id.clone();
5282 let prompts = self.state.config.prompts.clone();
5283 let language = self.state.config.graph.language.clone();
5284 let sessions = self.state.config.graph.sessions;
5285 let artifacts = agent::artifacts_dir(&self.state.dir());
5286 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5287 .findings
5288 .iter()
5289 .map(|f| Finding {
5290 id: f.id.clone(),
5291 severity: f.severity,
5292 file: f.file.clone(),
5293 line: f.line,
5294 title: f.title.clone(),
5295 detail: f.detail.clone(),
5296 })
5297 .collect();
5298 let fix_prompt = prompt::operator_fix_in(
5299 &self.state.instruction,
5300 &finding_list,
5301 reason,
5302 &stale_details,
5303 &head_at_request,
5304 &language,
5305 &self.state.config.graph.github_language,
5306 );
5307 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5308 drop(coordination);
5309 let (job, seat, out) = self
5310 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5311 prompt: fix_prompt.clone(),
5312 spec,
5313 seat,
5314 cwd: fix_worktree.clone(),
5315 timeout,
5316 allow_write: true,
5317 sessions,
5318 artifacts: artifacts.clone(),
5319 stem: "operator-fix".to_owned(),
5320 handover: None,
5321 })
5322 .await;
5323 let agent_id = seat.agent.clone();
5324
5325 let mut fix = FixRecord {
5326 agent: agent_id,
5327 addressed: Vec::new(),
5328 rejected: Vec::new(),
5329 notes: String::new(),
5330 committed: false,
5331 failed: None,
5332 duration_ms: 0,
5333 continuation: None,
5334 };
5335 let mut final_seat = seat.clone();
5336 match out {
5337 AgentOutcome::Ok(o) => {
5338 fix.duration_ms = o.duration_ms;
5339 let parsed = verdict::extract_json::<FixReport>(&o.text);
5340 let incomplete_reason = match &parsed {
5341 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5342 "the reply parsed, but it reported a command whose own CLI \
5343 never confirmed an exit status"
5344 .to_owned(),
5345 ),
5346 Ok(_) => None,
5347 Err(e) => Some(e.to_string()),
5348 };
5349 match incomplete_reason {
5350 None => {
5351 let report = parsed.expect("checked Ok above");
5352 fix.addressed = report.addressed;
5353 fix.rejected = report.rejected;
5354 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5355 }
5356 Some(reason) => {
5357 let (resumed_seat, resolved, failure, cont) = self
5358 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5359 .await;
5360 fix.duration_ms += cont.cumulative_wait_ms;
5361 fix.continuation = Some(cont);
5362 final_seat = resumed_seat;
5363 match resolved {
5364 Some(report) => {
5365 fix.addressed = report.addressed;
5366 fix.rejected = report.rejected;
5367 fix.notes =
5368 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5369 }
5370 None => fix.failed = failure,
5371 }
5372 }
5373 }
5374 }
5375 AgentOutcome::Dropped(o) => {
5376 fix.duration_ms = o.duration_ms;
5377 let why = o
5378 .dropped
5379 .as_ref()
5380 .map(|d| d.why.as_str())
5381 .unwrap_or("the CLI ended the stream without delivering its answer");
5382 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5383 }
5384 AgentOutcome::Quota(o) => {
5385 self.state.quota.push(QuotaLoss {
5386 seat: final_seat.key.clone(),
5387 node: "fix".to_owned(),
5388 at: Timestamp::now(),
5389 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5390 });
5391 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5392 }
5393 AgentOutcome::Failed(e) => fix.failed = Some(e),
5394 AgentOutcome::ResourceBlocked(e) => {
5395 fix.failed = Some(format!("the fixer was not started: {e}"));
5396 }
5397 }
5398 if fix.continuation.is_none() {
5399 fix.continuation = Some(ContinuationRecord::not_needed());
5400 }
5401 self.state.seats.insert(final_seat.key.clone(), final_seat);
5402
5403 let rescue_message = format!(
5404 "magi: operator-selected fix ({}) (uncommitted work)",
5405 self.state.operator_fixes[request_index]
5406 .findings
5407 .iter()
5408 .map(|f| f.id.as_str())
5409 .collect::<Vec<_>>()
5410 .join(", ")
5411 );
5412 let repo = self.state.repo.clone();
5413 let coordination = self.coordinate(&repo).await?;
5414 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5415 self.state.note_withheld("fix", &r.withheld);
5416 }
5417 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5418 fix.committed = after != head_at_request;
5419 git::worktree_remove(&self.state.repo, &fix_worktree)
5420 .await
5421 .ok();
5422 drop(coordination);
5423
5424 self.state.event(
5425 "fix",
5426 match &fix.failed {
5427 Some(reason) => format!(
5428 "operator fix: adoption report was lost ({reason}); {}",
5429 if fix.committed {
5430 "committed"
5431 } else {
5432 "NO new commit"
5433 }
5434 ),
5435 None => format!(
5436 "operator fix: {} addressed, {} rejected, {}",
5437 fix.addressed.len(),
5438 fix.rejected.len(),
5439 if fix.committed {
5440 "committed"
5441 } else {
5442 "NO new commit"
5443 }
5444 ),
5445 },
5446 );
5447
5448 for f in &mut self.state.operator_fixes[request_index].findings {
5455 f.outcome = if fix.failed.is_some() {
5456 OperatorFixOutcome::Unreported
5457 } else if fix.addressed.contains(&f.id) {
5458 OperatorFixOutcome::Addressed
5459 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5460 OperatorFixOutcome::Rejected { why: r.why.clone() }
5461 } else {
5462 OperatorFixOutcome::Unreported
5463 };
5464 }
5465
5466 let committed = fix.committed;
5467 if committed {
5468 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5469 }
5470 self.state.operator_fixes[request_index].fix = Some(fix);
5471 self.state.save()?;
5474
5475 if committed {
5476 self.state.event(
5477 "fix",
5478 format!(
5479 "operator fix committed {}; opening a follow-up review-only run",
5480 short(&after)
5481 ),
5482 );
5483 let origin =
5486 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5487 match Self::review(
5488 &self.state.repo,
5489 &winner.branch,
5490 self.state.config.clone(),
5491 origin,
5492 )
5493 .await
5494 {
5495 Ok(mut follow_up) => {
5496 follow_up.state.event(
5497 "start",
5498 format!(
5499 "requested by an operator fix on run {} for finding(s) {}",
5500 self.state.id,
5501 self.state.operator_fixes[request_index]
5502 .findings
5503 .iter()
5504 .map(|f| f.id.as_str())
5505 .collect::<Vec<_>>()
5506 .join(", "),
5507 ),
5508 );
5509 follow_up.state.save()?;
5510 let follow_up_id = follow_up.state.id.clone();
5511 let adopted = match crate::direct::adopt(
5514 &follow_up.state,
5515 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5516 ) {
5517 Ok(a) => a,
5518 Err(e) => {
5519 self.state.event(
5522 "fix",
5523 format!(
5524 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5525 ),
5526 );
5527 self.state.save()?;
5528 return Ok(());
5529 }
5530 };
5531 let executed = follow_up.execute().await;
5532 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5533 if let Some(a) = adopted {
5534 a.finish(&follow_up.state, executed);
5535 }
5536 if let Some(e) = failure {
5537 self.state.event(
5538 "fix",
5539 format!(
5540 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5541 ),
5542 );
5543 }
5544 self.state.operator_fixes[request_index].follow_up_review_run =
5545 Some(follow_up_id);
5546 }
5547 Err(e) => {
5548 self.state.event(
5549 "fix",
5550 format!("committed the fix but could not open a follow-up review: {e:#}"),
5551 );
5552 }
5553 }
5554 self.state.save()?;
5555 }
5556
5557 Ok(())
5558 }
5559
5560 async fn ask_fixer(
5580 &mut self,
5581 winner: &Candidate,
5582 node: &'static str,
5583 round: Option<usize>,
5584 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5585 ) -> (SeatJob, SeatState, AgentOutcome) {
5586 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5587 self.ask_fixer_among(attempts, node, round, None, build)
5588 .await
5589 }
5590
5591 async fn ask_fixer_among(
5596 &mut self,
5597 attempts: Vec<(AgentSpec, String)>,
5598 node: &'static str,
5599 round: Option<usize>,
5600 restore: Option<(&Path, &str)>,
5601 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5602 ) -> (SeatJob, SeatState, AgentOutcome) {
5603 let run_id = self.state.id.clone();
5604 let prompts = self.state.config.prompts.clone();
5605 let cache = self.state.config.cache_dir();
5606 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5607 let mut last = None;
5608 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5609 let seat = self.seat(&key, &spec.id);
5610 let mut job = build(spec.clone(), seat);
5611 if i > 0 {
5612 job.stem = format!("{}-{}", job.stem, spec.id);
5613 }
5614 let ctx = WaveCtx {
5615 carry_seats: false,
5616 pauses: Some((&self.pause, &self.interrupt)),
5617 run: &run_id,
5618 node,
5619 prompts: &prompts,
5620 cache: cache.as_deref(),
5621 round,
5622 };
5623 let (seat, out) =
5624 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5625 let advances = match &out {
5628 AgentOutcome::Ok(o) => agent::output_advances(o),
5629 AgentOutcome::ResourceBlocked(_) => false,
5630 _ => true,
5631 };
5632 if let Some(next) = ids.get(i + 1)
5633 && advances
5634 {
5635 self.state.seats.insert(seat.key.clone(), seat.clone());
5636 let class =
5637 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5638 let reason = match &out {
5639 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5640 other => fail_reason(other),
5641 };
5642 if let Some((wt, rev)) = restore
5646 && let Err(e) = self.graft_restore(wt, rev).await
5647 {
5648 let why =
5649 format!("could not restore the worktree before the next fixer: {e:#}");
5650 tracing::warn!("{why}");
5651 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5652 AgentOutcome::ResourceBlocked(why)
5653 } else {
5654 AgentOutcome::Failed(why)
5655 };
5656 last = Some((job, seat, out));
5657 break;
5658 }
5659 record_handover(
5660 &mut self.state,
5661 node,
5662 &seat.key,
5663 &spec.id,
5664 next,
5665 &class,
5666 &reason,
5667 );
5668 continue;
5669 }
5670 last = Some((job, seat, out));
5671 break;
5672 }
5673 last.expect("the fixer chain always has an entry")
5674 }
5675
5676 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5687 use crate::recovery::{Mode, Repair};
5688 if !self.state.reviews.is_empty() {
5689 return Ok(());
5690 }
5691 let Some(mut rec) = self.state.recovery.clone() else {
5692 return Ok(());
5693 };
5694 if rec.mode != Mode::VerifyRepair {
5695 return Ok(());
5696 }
5697 let wt = winner.worktree.clone();
5698 let mut head = git::rev_parse(&wt, "HEAD").await?;
5699 match rec.repair.clone() {
5700 Repair::Done { .. } => return Ok(()),
5701 Repair::WaitingForRestore { before } => {
5702 self.graft_restore(&wt, &before).await?;
5703 head = before;
5704 rec.repair = Repair::NotStarted;
5705 self.state.recovery = Some(rec.clone());
5706 self.state.save()?;
5707 }
5708 Repair::Started { before } => {
5709 if let Err(e) = self.graft_restore(&wt, &before).await {
5713 if e.is::<crate::repo_lock::ResourceBlocked>() {
5714 return Err(e);
5715 }
5716 return self
5717 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5718 .await;
5719 }
5720 rec.repair = Repair::Done {
5721 before: before.clone(),
5722 after: before,
5723 committed: false,
5724 failed: Some("interrupted; rolled back".to_owned()),
5725 };
5726 self.state
5727 .event("recovery", "the repair was interrupted and rolled back");
5728 self.state.recovery = Some(rec);
5729 self.state.save()?;
5730 return Ok(());
5731 }
5732 Repair::NotStarted => {}
5733 }
5734 if crate::recovery::recheck(&mut rec, &head) {
5735 self.state.event("recovery", rec.summary());
5736 self.state.recovery = Some(rec);
5737 self.state.save()?;
5738 return Ok(());
5739 }
5740 rec.repair = Repair::Started {
5741 before: head.clone(),
5742 };
5743 self.state.recovery = Some(rec.clone());
5744 self.state.save()?;
5745
5746 let language = self.state.config.graph.language.clone();
5747 let sessions = self.state.config.graph.sessions;
5748 let artifacts = agent::artifacts_dir(&self.state.dir());
5749 let fix_prompt = prompt::recovery_fix(
5750 &self.state.instruction,
5751 &rec,
5752 self.state.config.graph.review_rounds,
5753 &language,
5754 );
5755 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5756 self.state.event(
5757 "recovery",
5758 format!(
5759 "repair: the fixer starts from the failed verification on {}",
5760 short(&head)
5761 ),
5762 );
5763 let (_job, seat, out) = self
5764 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5765 prompt: fix_prompt.clone(),
5766 spec,
5767 seat,
5768 cwd: wt.clone(),
5769 timeout,
5770 allow_write: true,
5771 sessions,
5772 artifacts: artifacts.clone(),
5773 stem: "fix-recovery".to_owned(),
5774 handover: None,
5775 })
5776 .await;
5777 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5778 rec.repair = Repair::WaitingForRestore { before: head };
5779 self.state.recovery = Some(rec);
5780 return self.stop_for_resource();
5781 }
5782 let failed = match &out {
5783 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5784 AgentOutcome::Quota(o) => {
5785 self.state.quota.push(QuotaLoss {
5786 seat: seat.key.clone(),
5787 node: "fix".to_owned(),
5788 at: Timestamp::now(),
5789 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5790 });
5791 Some("rate limited (quota); the fixer could not run".to_owned())
5792 }
5793 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5794 AgentOutcome::Failed(e) => Some(e.clone()),
5795 };
5796 self.state.seats.insert(seat.key.clone(), seat);
5797 let coordination = self.coordinate(&wt).await?;
5798 let failed = match failed {
5799 Some(why) => {
5801 if let Err(e) = git::restore_to(&wt, &head).await {
5802 return self
5803 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5804 .await;
5805 }
5806 Some(why)
5807 }
5808 None => {
5809 if let Ok(r) = git::rescue_commit(
5810 &wt,
5811 "magi: recovery repair of the failed verification (uncommitted work)",
5812 )
5813 .await
5814 {
5815 self.state.note_withheld("fix", &r.withheld);
5816 }
5817 None
5818 }
5819 };
5820 let after = git::rev_parse(&wt, "HEAD").await?;
5821 drop(coordination);
5822 self.state.event(
5823 "recovery",
5824 match &failed {
5825 Some(why) => format!("repair: nothing kept ({why})"),
5826 None if after != head => {
5827 format!("repair: committed {}", short(&after))
5828 }
5829 None => "repair: the fixer made no commit".to_owned(),
5830 },
5831 );
5832 rec.repair = Repair::Done {
5833 before: head.clone(),
5834 committed: after != head,
5835 after,
5836 failed,
5837 };
5838 self.state.recovery = Some(rec);
5839 self.state.save()?;
5840 Ok(())
5841 }
5842
5843 async fn stop_recovery(
5846 &mut self,
5847 mut rec: crate::recovery::RecoveryRecord,
5848 head: &str,
5849 why: String,
5850 ) -> Result<()> {
5851 rec.repair = crate::recovery::Repair::Started {
5853 before: head.to_owned(),
5854 };
5855 self.state.recovery = Some(rec);
5856 self.state
5857 .event("recovery", format!("repair could not be undone: {why}"));
5858 self.state.status = RunStatus::Blocked;
5859 self.state.save()?;
5860 bail!("recovery repair could not be rolled back: {why}")
5861 }
5862
5863 async fn review_loop(&mut self) -> Result<()> {
5864 loop {
5865 if self
5866 .state
5867 .base_sync
5868 .as_ref()
5869 .is_some_and(|s| s.conflict.is_some())
5870 {
5871 return Ok(());
5872 }
5873 if self.catchup_repairable() {
5874 let record = self.state.reviews.last().expect("repairable round").clone();
5875 let winner = self
5876 .state
5877 .winner()
5878 .cloned()
5879 .context("catch-up repair has no winner")?;
5880 if !self.fix_review_round(&winner, record).await? {
5881 return Ok(());
5882 }
5883 }
5884 self.review_loop_pass().await?;
5885 if !self.catchup_repairable() {
5886 return Ok(());
5887 }
5888 }
5889 }
5890
5891 fn catchup_repairable(&self) -> bool {
5894 self.state.reviews.len() < self.state.config.graph.review_rounds
5895 && self.state.reviews.last().is_some_and(|r| {
5896 !r.clean
5897 && r.blocking == 0
5898 && r.fix.is_none()
5899 && r.e2e_status() == E2eStatus::Failed
5900 })
5901 }
5902
5903 async fn review_loop_pass(&mut self) -> Result<()> {
5904 if self
5909 .state
5910 .base_sync
5911 .as_ref()
5912 .is_some_and(|s| s.conflict.is_some())
5913 {
5914 return Ok(());
5915 }
5916 let run_id = self.state.id.clone();
5921 let prompts = self.state.config.prompts.clone();
5922 let Some(winner) = self.state.winner().cloned() else {
5923 return Ok(());
5924 };
5925 let max_rounds = self.state.config.graph.review_rounds;
5926 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5936 if status == RunStatus::Gating {
5939 self.record_contested_handoff();
5940 }
5941 self.state.status = status;
5942 self.state.save()?;
5943 return Ok(());
5944 }
5945 self.state.status = RunStatus::Reviewing;
5946 if self.state.reviews.last().is_some_and(|r| {
5956 r.e2e_status() == E2eStatus::ResourceBlocked
5957 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5958 }) {
5959 let shell = self.state.config.shell();
5960 return self
5961 .stop_unresolved_review(
5962 "the last round's own verification never resolved",
5963 &shell,
5964 &winner.worktree,
5965 )
5966 .await;
5967 }
5968
5969 self.recovery_repair(&winner).await?;
5970 if self.resource_halt()? {
5971 return Ok(());
5972 }
5973
5974 let repo = self.state.repo.clone();
5975 let root = self.state.worktree_root();
5976 let language = self.state.config.graph.language.clone();
5977 let sessions = self.state.config.graph.sessions;
5978 let artifacts = self.merge_repair_artifacts();
5979 let base = self.landing_base();
5980 let base_short = short(&base);
5981 let reviewers = self.roles.reviewers.clone();
5982 let shell = self.state.config.shell();
5983
5984 for round in (self.state.reviews.len() + 1)..=max_rounds {
5985 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5986 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5987 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5988 let prev_verification = self
5997 .state
5998 .reviews
5999 .last()
6000 .and_then(|r| r.verification_summary(&head));
6001 let prior_section = match (&self.state.recovery, round) {
6004 (Some(rec), 1) if self.state.reviews.is_empty() => {
6005 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6006 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6007 Some(crate::recovery::head_of(
6008 &git::diff(&winner.worktree, reviewed, "HEAD")
6009 .await
6010 .unwrap_or_default(),
6011 crate::recovery::SINCE_DIFF_MAX,
6012 ))
6013 }
6014 _ => None,
6015 };
6016 Some(prompt::recovery_review(rec, since.as_deref()))
6017 }
6018 _ => None,
6019 };
6020
6021 let mut jobs = Vec::new();
6025 let coordination = self.coordinate(&repo).await?;
6026 for (r, spec) in reviewers.iter().cloned().enumerate() {
6027 let wt = root.join(format!("review-{}", r + 1));
6028 if wt.exists() {
6031 git::reset_detached(&wt, &head).await?;
6032 } else {
6033 git::worktree_add_detached(&repo, &wt, &head).await?;
6034 }
6035 self.setup_seat_worktree(&repo, &wt, false).await?;
6036 let seat_key = format!("review-{}", r + 1);
6037 let spec = pick_start_spec(
6040 &self.roles.reviewer_roster,
6041 spec,
6042 self.state.seat_history.get(&seat_key),
6043 );
6044 let seat = self.seat(&seat_key, &spec.id);
6045 jobs.push(SeatJob {
6046 prompt: prompt::review(&prompt::ReviewCtx {
6047 instruction: &self.state.instruction,
6048 branch: &winner.branch,
6049 base_short: &base_short,
6050 stat: &stat,
6051 patch: &patch,
6052 verification: prev_verification.as_ref(),
6053 prior: prior_section.as_deref(),
6054 reviewers: reviewers.len(),
6055 round,
6056 rounds: max_rounds,
6057 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6060 lens: Lens::for_seat(r),
6061 language: &language,
6062 github_language: &self.state.config.graph.github_language,
6063 }),
6064 spec,
6065 seat,
6066 cwd: wt,
6067 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6068 allow_write: false,
6069 sessions,
6070 artifacts: artifacts.clone(),
6071 stem: format!("review-{round}-{}", r + 1),
6072 handover: None,
6073 });
6074 }
6075
6076 drop(coordination);
6077 self.state.event(
6078 "review",
6079 format!(
6080 "round {round}: {} reviewers on {}",
6081 jobs.len(),
6082 short(&head)
6083 ),
6084 );
6085 let mut quota_losses = Vec::new();
6086 let review_retries = self.state.config.graph.retries;
6087 let review_cache = self.state.config.cache_dir();
6088 let ctx = WaveCtx {
6089 carry_seats: true,
6090 pauses: Some((&self.pause, &self.interrupt)),
6091 run: &run_id,
6092 node: "review",
6093 prompts: &prompts,
6094 cache: review_cache.as_deref(),
6095 round: Some(round),
6096 };
6097 let results = ask_json_wave::<Review>(
6098 jobs,
6099 Arc::clone(&self.sem),
6100 review_retries,
6101 &self.roles.reviewer_roster,
6102 &ctx,
6103 &mut quota_losses,
6104 &mut self.state,
6105 &|_: &Review| Ok(()),
6106 )
6107 .await;
6108 if self.state.resource_block.is_some() {
6109 return self.stop_for_resource();
6110 }
6111 let round_quota_missing = quota_losses.len();
6115 self.state.quota.extend(quota_losses);
6116
6117 let mut records = Vec::new();
6118 let mut all_findings = Vec::new();
6119 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6120 let agent_id = seat.agent.clone();
6121 self.state.seats.insert(seat.key.clone(), seat);
6122 let mut record = ReviewRecord {
6123 reviewer: r + 1,
6124 agent: agent_id,
6125 summary: String::new(),
6126 findings: Vec::new(),
6127 vote: None,
6128 failed: None,
6129 duration_ms: 0,
6130 attempts,
6136 };
6137 match res {
6138 Ok((review, out)) => {
6139 record.summary =
6147 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6148 record.vote = Some(review.vote);
6149 record.duration_ms = out.duration_ms;
6150 for (n, mut f) in review.findings.into_iter().enumerate() {
6151 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6154 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6155 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6156 f.file = f
6162 .file
6163 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6164 all_findings.push(f.clone());
6165 record.findings.push(f);
6166 }
6167 self.state.event(
6168 "review",
6169 format!(
6170 "round {round}: reviewer {} voted {} with {} finding(s)",
6171 r + 1,
6172 review.vote.label(),
6173 record.findings.len()
6174 ),
6175 );
6176 }
6177 Err(e) => {
6178 record.failed = Some(e.to_string());
6179 self.state.event(
6180 "review",
6181 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6182 );
6183 }
6184 }
6185 records.push(record);
6186 }
6187
6188 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6195 let vote_split =
6196 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6197 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6198 if vote_split {
6199 self.state.event(
6200 "review",
6201 format!(
6202 "round {round}: votes split ({}) — one round of reconsideration",
6203 initial_votes
6204 .iter()
6205 .map(|v| v.label())
6206 .collect::<Vec<_>>()
6207 .join(", ")
6208 ),
6209 );
6210 let panel: Vec<ReviewSeatReport<'_>> = records
6213 .iter()
6214 .filter_map(|r| {
6215 r.vote.map(|vote| ReviewSeatReport {
6216 reviewer: r.reviewer,
6217 vote,
6218 summary: &r.summary,
6219 findings: &r.findings,
6220 })
6221 })
6222 .collect();
6223
6224 let mut jobs = Vec::new();
6225 let mut seats_at = Vec::new();
6226 for (r, spec) in reviewers.iter().cloned().enumerate() {
6227 if records[r].vote.is_none() {
6231 continue;
6232 }
6233 let wt = root.join(format!("review-{}", r + 1));
6234 let seat_key = format!("review-{}", r + 1);
6235 let spec = self.occupant(&seat_key, spec);
6236 let seat = self.seat(&seat_key, &spec.id);
6237 let build = |with_patch: bool| {
6245 prompt::review_reconsider(&ReviewReconsiderCtx {
6246 instruction: &self.state.instruction,
6247 reviewer: r + 1,
6248 lens: Lens::for_seat(r),
6249 panel: &panel,
6250 patch: with_patch.then_some(ReviewPatch {
6251 branch: &winner.branch,
6252 base_short: &base_short,
6253 stat: &stat,
6254 patch: &patch,
6255 }),
6256 round,
6257 rounds: max_rounds,
6258 language: &language,
6259 })
6260 };
6261 let full = build(true);
6262 let prompt = if has_context(&spec, &seat, sessions) {
6263 build(false)
6264 } else {
6265 full.clone()
6266 };
6267 jobs.push(SeatJob {
6268 prompt,
6269 spec,
6270 seat,
6271 cwd: wt,
6272 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6273 allow_write: false,
6274 sessions,
6275 artifacts: artifacts.clone(),
6276 stem: format!("review-{round}-reconsider-{}", r + 1),
6277 handover: Some(full),
6278 });
6279 seats_at.push(r);
6280 }
6281
6282 let mut recon_quota_losses = Vec::new();
6283 let recon_cache = self.state.config.cache_dir();
6284 let recon_ctx = WaveCtx {
6285 carry_seats: true,
6286 pauses: Some((&self.pause, &self.interrupt)),
6287 run: &run_id,
6288 node: "review",
6289 prompts: &prompts,
6290 cache: recon_cache.as_deref(),
6291 round: Some(round),
6292 };
6293 let recon_results = ask_json_wave::<ReviewRevote>(
6294 jobs,
6295 Arc::clone(&self.sem),
6296 review_retries,
6297 &self.roles.reviewer_roster,
6298 &recon_ctx,
6299 &mut recon_quota_losses,
6300 &mut self.state,
6301 &|_: &ReviewRevote| Ok(()),
6302 )
6303 .await;
6304 if self.state.resource_block.is_some() {
6305 return self.stop_for_resource();
6306 }
6307 self.state.quota.extend(recon_quota_losses);
6308
6309 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6310 let agent_id = seat.agent.clone();
6311 self.state.seats.insert(seat.key.clone(), seat);
6312 let mut rec = ReviewRevoteRecord {
6313 reviewer: r + 1,
6314 agent: agent_id,
6315 vote: None,
6316 reason: String::new(),
6317 failed: None,
6318 };
6319 match res {
6320 Ok((rv, _)) => {
6321 rec.vote = Some(rv.vote);
6322 rec.reason =
6323 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6324 self.state.event(
6325 "review",
6326 format!(
6327 "round {round}: reviewer {} revoted {}",
6328 r + 1,
6329 rv.vote.label()
6330 ),
6331 );
6332 }
6333 Err(e) => {
6334 rec.failed = Some(e.to_string());
6335 self.state.event(
6336 "review",
6337 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6338 );
6339 }
6340 }
6341 reconsideration.push(rec);
6342 }
6343 } else if initial_votes.len() > 1 {
6344 self.state.event(
6345 "review",
6346 format!(
6347 "round {round}: votes agreed ({}) — no reconsideration",
6348 initial_votes[0].label()
6349 ),
6350 );
6351 }
6352
6353 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6354 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6355 let defer_e2e =
6366 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6367 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6368 let reason =
6369 format!("{blocking} blocking finding(s) already required a fix this round");
6370 self.state.event(
6371 "verify",
6372 format!(
6373 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6374 {}); it will run once a round has none left",
6375 short(&head)
6376 ),
6377 );
6378 (Vec::new(), false, true, Some(reason))
6379 } else {
6380 let e2e_commands = self.state.config.verify.e2e.clone();
6381 let cache_dir = self.state.config.cache_dir();
6382 let context = format!("round {round}");
6383 let wait = self.repository_wait();
6384 let (e2e, verify_retried) = with_cache_lease(
6385 &mut self.state,
6386 cache_dir.as_deref(),
6387 "e2e",
6388 "e2e",
6389 &winner.worktree,
6390 &head,
6391 verify_timeout,
6392 &context,
6393 &wait,
6394 |state, budget| {
6395 let shell = shell.clone();
6396 let e2e_commands = e2e_commands.clone();
6397 let worktree = winner.worktree.clone();
6398 let context = context.clone();
6399 async move {
6400 run_e2e_with_retry(
6401 state,
6402 &shell,
6403 &e2e_commands,
6404 &worktree,
6405 budget,
6406 &context,
6407 )
6408 .await
6409 }
6410 },
6411 )
6412 .await?;
6413 (e2e, verify_retried, false, None)
6414 };
6415
6416 let expected = records.len();
6417 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6418 let incomplete = answered < expected;
6419 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6420 let policy = self.state.config.graph.incomplete_review;
6421 let clean = round_is_clean(
6422 blocking,
6423 e2e_ok,
6424 answered,
6425 expected,
6426 round_quota_missing,
6427 policy,
6428 );
6429
6430 let mut round_record = ReviewRound {
6431 round,
6432 head: head.clone(),
6433 verified_head: None,
6434 verified_at: None,
6435 reviews: records,
6436 e2e,
6437 verify_retried,
6438 e2e_deferred,
6439 e2e_defer_reason,
6440 fix: None,
6441 blocking,
6442 answered,
6443 expected,
6444 clean,
6445 progressed: false,
6446 vote_split,
6447 reconsideration,
6448 verdict: None,
6449 };
6450 round_record.verdict = ReviewVote::worst(
6454 round_record
6455 .final_votes()
6456 .into_iter()
6457 .map(|(_, _, vote)| vote),
6458 );
6459 if !matches!(
6470 round_record.e2e_status(),
6471 E2eStatus::Deferred | E2eStatus::NotConfigured
6472 ) {
6473 round_record.verified_head = Some(head.clone());
6474 round_record.verified_at = Some(Timestamp::now());
6475 }
6476
6477 if incomplete {
6478 let missing: Vec<String> = round_record
6479 .reviews
6480 .iter()
6481 .filter(|r| r.failed.is_some())
6482 .map(|r| format!("review-{}", r.reviewer))
6483 .collect();
6484 self.state.event(
6485 "review",
6486 format!(
6487 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6488 missing.join(", ")
6489 ),
6490 );
6491 }
6492
6493 if clean {
6494 self.state.event(
6495 "review",
6496 if incomplete && policy == IncompleteReviewPolicy::Warn {
6497 format!(
6498 "round {round}: clean (warn policy, incomplete panel) — no \
6499 blocking findings from the seats that answered, verification green"
6500 )
6501 } else if incomplete {
6502 format!(
6503 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6504 quorum) — no blocking findings from the seats that answered, \
6505 verification green",
6506 expected - answered
6507 )
6508 } else {
6509 format!("round {round}: clean — no blocking findings, verification green")
6510 },
6511 );
6512 self.state.reviews.push(round_record);
6513 self.state.status = RunStatus::Gating;
6514 self.state.save()?;
6515 return Ok(());
6516 }
6517
6518 if incomplete && blocking == 0 && e2e_ok {
6526 self.state.reviews.push(round_record);
6527 self.state.save()?;
6528 if round == max_rounds {
6529 self.state.status = RunStatus::Blocked;
6530 self.state.event(
6531 "review",
6532 format!(
6533 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6534 refusing to call it clean",
6535 expected - answered
6536 ),
6537 );
6538 return Ok(());
6539 }
6540 continue;
6541 }
6542
6543 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6555 self.state.reviews.push(round_record);
6556 return self
6557 .stop_unresolved_review(
6558 "the round's own verification could not run",
6559 &shell,
6560 &winner.worktree,
6561 )
6562 .await;
6563 }
6564
6565 if round == max_rounds {
6566 self.state.reviews.push(round_record);
6567 return self
6568 .stop_reviewing(
6569 &format!(
6570 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6571 ),
6572 &shell,
6573 &winner.worktree,
6574 )
6575 .await;
6576 }
6577
6578 if !self.fix_review_round(&winner, round_record).await? {
6579 return Ok(());
6580 }
6581 }
6582 Ok(())
6583 }
6584
6585 async fn fix_review_round(
6588 &mut self,
6589 winner: &Candidate,
6590 mut round_record: ReviewRound,
6591 ) -> Result<bool> {
6592 let round = round_record.round;
6593 let shell = self.state.config.shell();
6594 let step = self.run_fix_round(winner, &mut round_record).await?;
6595 if matches!(step, FixStep::Resource) {
6596 self.stop_for_resource()?;
6597 return Ok(false);
6598 }
6599 if self.state.reviews.len() < round {
6600 self.state.reviews.push(round_record);
6601 } else {
6602 self.state.reviews[round - 1] = round_record;
6603 }
6604 self.state.save()?;
6605
6606 if matches!(step, FixStep::Recorded { report_lost: true }) {
6619 self.stop_reviewing(
6620 "the fixer's adoption report never came back, even after resuming its \
6621 own seat; refusing to start another round against the same worktree \
6622 while that is unresolved",
6623 &shell,
6624 &winner.worktree,
6625 )
6626 .await?;
6627 return Ok(false);
6628 }
6629
6630 let streak = self
6631 .state
6632 .reviews
6633 .iter()
6634 .rev()
6635 .take_while(|r| !r.progressed)
6636 .count();
6637 if streak >= STAGNANT_LIMIT {
6638 self.stop_reviewing(
6639 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6640 &shell,
6641 &winner.worktree,
6642 )
6643 .await?;
6644 return Ok(false);
6645 }
6646 Ok(true)
6647 }
6648
6649 async fn run_fix_round(
6651 &mut self,
6652 winner: &Candidate,
6653 round_record: &mut ReviewRound,
6654 ) -> Result<FixStep> {
6655 let round = round_record.round;
6656 let max_rounds = self.state.config.graph.review_rounds;
6657 let run_id = self.state.id.clone();
6658 let prompts = self.state.config.prompts.clone();
6659 let language = self.state.config.graph.language.clone();
6660 let sessions = self.state.config.graph.sessions;
6661 let artifacts = self.merge_repair_artifacts();
6662 let base = self.landing_base();
6663 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6664 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6665 let this_round_verification = round_record.verification_summary(&head);
6666 let all_findings: Vec<_> = round_record
6667 .reviews
6668 .iter()
6669 .flat_map(|r| r.findings.iter().cloned())
6670 .collect();
6671 let blocking_findings: Vec<_> = all_findings
6674 .iter()
6675 .filter(|f| f.severity.blocks())
6676 .cloned()
6677 .collect();
6678 let fix_prompt = prompt::fix_in(
6679 &self.state.instruction,
6680 &blocking_findings,
6681 this_round_verification.as_ref(),
6682 round,
6683 max_rounds,
6684 &language,
6685 &self.state.config.graph.github_language,
6686 );
6687 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6688 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6689 let (job, seat, out) = self
6690 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6691 prompt: fix_prompt.clone(),
6692 spec,
6693 seat,
6694 cwd: winner.worktree.clone(),
6695 timeout,
6696 allow_write: true,
6697 sessions,
6698 artifacts: artifacts.clone(),
6699 stem: format!("fix-{round}"),
6700 handover: None,
6701 })
6702 .await;
6703 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6706 return Ok(FixStep::Resource);
6707 }
6708 let agent_id = seat.agent.clone();
6709
6710 let mut fix = FixRecord {
6711 agent: agent_id,
6712 addressed: Vec::new(),
6713 rejected: Vec::new(),
6714 notes: String::new(),
6715 committed: false,
6716 failed: None,
6717 duration_ms: 0,
6718 continuation: None,
6719 };
6720 let mut continuation = ContinuationRecord::not_needed();
6721 let mut final_seat = seat.clone();
6722 match out {
6723 AgentOutcome::Ok(o) => {
6724 fix.duration_ms = o.duration_ms;
6725 let parsed = verdict::extract_json::<FixReport>(&o.text);
6726 let incomplete_reason = match &parsed {
6733 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6734 "the reply parsed, but it reported a command whose own CLI never \
6735 confirmed an exit status"
6736 .to_owned(),
6737 ),
6738 Ok(_) => None,
6739 Err(e) => Some(e.to_string()),
6740 };
6741 match incomplete_reason {
6742 None => {
6743 let report = parsed.expect("checked Ok above");
6744 fix.addressed = report.addressed;
6745 fix.rejected = report.rejected;
6746 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6747 }
6748 Some(reason) => {
6749 let (resumed_seat, resolved, failure, cont) = self
6750 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6751 .await;
6752 fix.duration_ms += cont.cumulative_wait_ms;
6753 continuation = cont;
6754 final_seat = resumed_seat;
6755 match resolved {
6756 Some(report) => {
6757 fix.addressed = report.addressed;
6758 fix.rejected = report.rejected;
6759 fix.notes =
6760 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6761 }
6762 None => fix.failed = failure,
6763 }
6764 }
6765 }
6766 }
6767 AgentOutcome::Dropped(o) => {
6769 fix.duration_ms = o.duration_ms;
6770 let why = o
6771 .dropped
6772 .as_ref()
6773 .map(|d| d.why.as_str())
6774 .unwrap_or("the CLI ended the stream without delivering its answer");
6775 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6776 }
6777 AgentOutcome::Quota(o) => {
6778 self.state.quota.push(QuotaLoss {
6779 seat: final_seat.key.clone(),
6780 node: "fix".to_owned(),
6781 at: Timestamp::now(),
6782 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6783 });
6784 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6785 }
6786 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6787 }
6788 fix.continuation = Some(continuation);
6789 self.state.seats.insert(final_seat.key.clone(), final_seat);
6790 let coordination = self.coordinate(&winner.worktree).await?;
6791 if let Ok(r) = git::rescue_commit(
6792 &winner.worktree,
6793 &format!("magi: review round {round} fixes (uncommitted work)"),
6794 )
6795 .await
6796 {
6797 self.state.note_withheld("fix", &r.withheld);
6798 }
6799 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6800 fix.committed = after != before;
6801 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6809 drop(coordination);
6810 let progressed = diff_after != patch;
6811 let commit_note = if fix.committed {
6812 "committed"
6813 } else {
6814 "NO new commit"
6815 };
6816 let tree_note = if progressed {
6817 "changed vs base"
6818 } else {
6819 "unchanged vs base"
6820 };
6821 self.state.event(
6822 "fix",
6823 match &fix.failed {
6824 Some(reason) => {
6830 format!(
6831 "round {round}: fixer's adoption report was lost ({reason}); \
6832 {commit_note}, tree {tree_note}"
6833 )
6834 }
6835 None => format!(
6836 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6837 {tree_note}{}",
6838 fix.addressed.len(),
6839 fix.rejected.len(),
6840 if continuation.outcome == ContinuationOutcome::Resumed {
6841 format!(
6842 " (adoption report recovered after {} continuation(s))",
6843 continuation.attempts
6844 )
6845 } else {
6846 String::new()
6847 },
6848 ),
6849 },
6850 );
6851 round_record.fix = Some(fix);
6852 round_record.progressed = progressed;
6853 Ok(FixStep::Recorded {
6854 report_lost: matches!(
6855 continuation.outcome,
6856 ContinuationOutcome::Exhausted
6857 | ContinuationOutcome::QuotaLost
6858 | ContinuationOutcome::NoSession
6859 ),
6860 })
6861 }
6862
6863 fn record_contested_handoff(&mut self) {
6897 if self.state.contested_handoff.is_some() {
6898 return;
6899 }
6900 let Some(contested) = self
6901 .state
6902 .reviews
6903 .last()
6904 .and_then(ReviewRound::contested_handoff)
6905 else {
6906 return;
6907 };
6908 self.state.event(
6909 "review",
6910 format!(
6911 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6912 wait for the owner's approval",
6913 contested.findings.len(),
6914 contested.rejecters.len()
6915 ),
6916 );
6917 self.state.contested_handoff = Some(contested);
6918 }
6919
6920 async fn stop_unresolved_review(
6921 &mut self,
6922 why: &str,
6923 shell: &[String],
6924 worktree: &Path,
6925 ) -> Result<()> {
6926 self.stop_reviewing_inner(why, shell, worktree, true).await
6927 }
6928
6929 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6930 self.stop_reviewing_inner(why, shell, worktree, false).await
6931 }
6932
6933 async fn stop_reviewing_inner(
6934 &mut self,
6935 why: &str,
6936 shell: &[String],
6937 worktree: &Path,
6938 repairable: bool,
6939 ) -> Result<()> {
6940 let round_idx = self.state.reviews.len() - 1;
6941 let needs_catchup_run = matches!(
6949 self.state.reviews[round_idx].e2e_status(),
6950 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6951 );
6952 if needs_catchup_run {
6953 let round = self.state.reviews[round_idx].round;
6954 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6955 let commands = self.state.config.verify.e2e.clone();
6956 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6957 let cache_dir = self.state.config.cache_dir();
6958 let context = format!(
6959 "round {round}: verification unresolved, catching up before the final decision"
6960 );
6961 let wait = self.repository_wait();
6962 let (outcomes, verify_retried) = with_cache_lease(
6963 &mut self.state,
6964 cache_dir.as_deref(),
6965 "e2e",
6966 "e2e",
6967 worktree,
6968 &attempted_head,
6969 timeout,
6970 &context,
6971 &wait,
6972 |state, budget| {
6973 let shell = shell.to_vec();
6974 let commands = commands.clone();
6975 let context = context.clone();
6976 async move {
6977 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6978 .await
6979 }
6980 },
6981 )
6982 .await?;
6983 let last = &mut self.state.reviews[round_idx];
6984 last.e2e = outcomes;
6985 last.verify_retried = verify_retried;
6986 last.verified_head = Some(attempted_head);
6993 last.verified_at = Some(Timestamp::now());
6994 if verify_inconclusive(&last.e2e) {
6995 self.state.save()?;
7002 return Ok(());
7003 }
7004 last.e2e_deferred = false;
7005 }
7006 let last = &self.state.reviews[round_idx];
7007 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7008
7009 match last.e2e_status() {
7010 E2eStatus::Failed => {
7011 let red: Vec<String> = last
7012 .e2e
7013 .iter()
7014 .filter(|o| !o.ok())
7015 .map(|o| {
7016 format!(
7017 "`{}` -> {:?}\n{}",
7018 o.command,
7019 o.code,
7020 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7021 )
7022 })
7023 .collect();
7024 let can_repair = last.blocking == 0 && last.fix.is_none();
7025 self.state.reviews[round_idx].clean = false;
7026 self.state
7027 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7028 self.state.status = RunStatus::Blocked;
7029 if repairable
7030 && can_repair
7031 && self.state.reviews.len() < self.state.config.graph.review_rounds
7032 {
7033 self.state.status = RunStatus::Reviewing;
7034 }
7035 }
7036 E2eStatus::ResourceBlocked => {
7041 self.state.event(
7042 "review",
7043 format!(
7044 "{why}; e2e could not run (shared build cache unavailable); not \
7045 deciding yet"
7046 ),
7047 );
7048 }
7049 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7050 self.state.event(
7051 "review",
7052 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7053 );
7054 self.record_contested_handoff();
7055 self.state.status = RunStatus::Gating;
7056 }
7057 }
7058 self.state.save()?;
7059 Ok(())
7060 }
7061
7062 async fn gate(&mut self) -> Result<()> {
7065 if self.state.status == RunStatus::Failed
7077 || self
7078 .state
7079 .base_sync
7080 .as_ref()
7081 .is_some_and(|s| s.conflict.is_some())
7082 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7083 != Some(RunStatus::Gating)
7084 {
7085 return Ok(());
7086 }
7087 if self.state.gate_ran {
7088 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7099 self.state.status = RunStatus::Blocked;
7100 self.state.save()?;
7101 }
7102 return Ok(());
7103 }
7104 let Some(winner) = self.state.winner().cloned() else {
7105 return Ok(());
7106 };
7107 self.state.status = RunStatus::Gating;
7108 let mut outcomes = self.run_gate(&winner).await?;
7109 loop {
7110 if verify_inconclusive(&outcomes) {
7121 self.state.save()?;
7122 return Ok(());
7123 }
7124 if outcomes.iter().all(CommandOutcome::ok) {
7125 break;
7126 }
7127 match self.gate_fix_round(&winner, &outcomes).await? {
7128 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7129 GateFix::Stop => break,
7130 GateFix::Defer => {
7131 self.state.save()?;
7132 return Ok(());
7133 }
7134 }
7135 }
7136 let passed = outcomes.iter().all(CommandOutcome::ok);
7137 self.state.gate = outcomes;
7138 self.state.gate_ran = true;
7139 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7140 if !passed {
7141 self.state.status = RunStatus::Blocked;
7142 let spent = self.state.gate_fixes.len();
7143 self.state.event(
7144 "gate",
7145 if spent == 0 {
7146 "gate failed; not merging".to_owned()
7147 } else {
7148 format!("gate failed after {spent} gate-fix round(s); not merging")
7149 },
7150 );
7151 }
7152 self.state.save()?;
7153 Ok(())
7154 }
7155
7156 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7166 let commands = state.config.verify.pre_gate.clone();
7167 if commands.is_empty() {
7168 return;
7169 }
7170 let shell = state.config.shell();
7171 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7172 let (outcomes, _) = run_commands(
7173 state,
7174 "pre_gate",
7175 "pre_gate",
7176 0,
7177 &shell,
7178 &commands,
7179 &winner.worktree,
7180 timeout,
7181 )
7182 .await;
7183 for o in &outcomes {
7184 if !o.ok() {
7185 tracing::warn!(
7186 "pre_gate `{}` failed ({:?}); the gate decides",
7187 o.command,
7188 o.code
7189 );
7190 }
7191 state.event(
7192 "pre_gate",
7193 format!(
7194 "`{}` -> {}",
7195 o.command,
7196 if o.ok() {
7197 "pass".to_owned()
7198 } else {
7199 format!(
7200 "FAIL ({:?})\n{}",
7201 o.code,
7202 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7203 )
7204 }
7205 ),
7206 );
7207 }
7208 state.pre_gate = outcomes;
7209 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7210 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7211 Ok(head) => {
7212 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7213 state.pre_gate_commit = Some(head);
7214 }
7215 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7216 },
7217 Ok(false) => {}
7218 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7219 }
7220 if let Err(e) = state.save() {
7221 tracing::warn!("could not persist the pre_gate record: {e:#}");
7222 }
7223 }
7224
7225 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7228 let wait = self.repository_wait();
7229 run_landing_gate(&mut self.state, winner, &wait).await
7230 }
7231
7232 async fn gate_fix_round(
7245 &mut self,
7246 winner: &Candidate,
7247 outcomes: &[CommandOutcome],
7248 ) -> Result<GateFix> {
7249 let cap = self.state.config.graph.gate_fix_rounds;
7250 let spent = self.state.gate_fixes.len();
7251 if spent >= cap {
7252 if cap > 0 {
7253 self.state.event(
7254 "gate",
7255 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7256 );
7257 }
7258 return Ok(GateFix::Stop);
7259 }
7260 if !gate_fixable(outcomes) {
7261 self.state.event(
7262 "gate",
7263 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7264 command or similar); not spending a fix round on it",
7265 );
7266 return Ok(GateFix::Stop);
7267 }
7268 let min_free = self.state.config.disk.min_free_bytes;
7269 if min_free > 0 {
7270 match crate::disk::free_bytes(&winner.worktree) {
7271 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7272 Ok(free) => {
7273 self.state.event(
7274 "gate",
7275 format!(
7276 "only {free} bytes free ({min_free} required by `[disk] \
7277 min_free_bytes`); not spending a fix round on a failure the disk \
7278 may explain"
7279 ),
7280 );
7281 return Ok(GateFix::Stop);
7282 }
7283 Err(e) => {
7284 self.state.event(
7285 "gate",
7286 format!("free disk space could not be measured ({e:#}); no fix round"),
7287 );
7288 return Ok(GateFix::Stop);
7289 }
7290 }
7291 }
7292
7293 let attempt = spent + 1;
7294 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7295 let base = self.landing_base();
7296 let fix_prompt = prompt::gate_fix_in(
7297 &self.state.instruction,
7298 &failed,
7299 attempt,
7300 cap,
7301 &self.state.config.graph.language,
7302 &self.state.config.graph.github_language,
7303 );
7304 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7305 let sessions = self.state.config.graph.sessions;
7306 let artifacts = self.merge_repair_artifacts();
7307 self.state.event(
7308 "gate",
7309 format!("gate failed; gate-fix round {attempt} of {cap}"),
7310 );
7311 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7312 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7313 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7316 head: before.clone(),
7317 passed: false,
7318 outcomes: Vec::new(),
7319 });
7320 self.state.save()?;
7321 let (_, seat, out) = self
7322 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7323 prompt: fix_prompt.clone(),
7324 spec,
7325 seat,
7326 cwd: winner.worktree.clone(),
7327 timeout,
7328 allow_write: true,
7329 sessions,
7330 artifacts: artifacts.clone(),
7331 stem: format!("gate-fix-{attempt}"),
7332 handover: None,
7333 })
7334 .await;
7335 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7338 return Ok(GateFix::Defer);
7339 }
7340 let mut record = GateFixRecord {
7341 agent: seat.agent.clone(),
7342 failed,
7343 notes: String::new(),
7344 committed: false,
7345 error: None,
7346 };
7347 match out {
7348 AgentOutcome::Ok(o) => {
7349 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7352 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7353 }
7354 }
7355 AgentOutcome::Dropped(_) => {
7356 record.error = Some("the CLI dropped the stream".to_owned());
7357 }
7358 AgentOutcome::Quota(o) => {
7359 self.state.quota.push(QuotaLoss {
7360 seat: seat.key.clone(),
7361 node: "gate-fix".to_owned(),
7362 at: Timestamp::now(),
7363 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7364 });
7365 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7366 }
7367 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7368 }
7369 self.state.seats.insert(seat.key.clone(), seat);
7370 let coordination = self.coordinate(&winner.worktree).await?;
7371 if let Ok(r) = git::rescue_commit(
7372 &winner.worktree,
7373 &format!("magi: gate fix {attempt} (uncommitted work)"),
7374 )
7375 .await
7376 {
7377 self.state.note_withheld("gate-fix", &r.withheld);
7378 }
7379 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7380 record.committed = after != before;
7381 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7382 let note = record.error.clone();
7383 self.state.gate_fixes.push(record);
7384 self.state.save()?;
7385 if !changed {
7386 self.state.event(
7387 "gate",
7388 match note {
7389 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7390 None => format!("gate-fix round {attempt}: the tree did not change"),
7391 },
7392 );
7393 return Ok(GateFix::Stop);
7394 }
7395 self.state.event(
7396 "gate",
7397 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7398 );
7399
7400 drop(coordination);
7401 let wait = self.repository_wait();
7402 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7403 }
7404
7405 async fn guarded_pr_message(
7412 &mut self,
7413 winner: &Candidate,
7414 facts: Option<&BranchFacts>,
7415 posting: bool,
7416 ) -> Result<Option<PrMessage>> {
7417 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7418 let head = match final_head.as_deref() {
7419 Some(h) => crate::pr_outcome::FinalHead::At(h),
7420 None => crate::pr_outcome::FinalHead::Unknown,
7421 };
7422 let summarized = if posting {
7424 Some(self.summarize_change(winner, final_head.as_deref()).await)
7425 } else {
7426 None
7427 };
7428 if self.state.resource_block.is_some() {
7433 self.state.event(
7434 "pr-summary",
7435 "a shared resource was busy; nothing was published, resume to summarize again",
7436 );
7437 return Ok(None);
7438 }
7439 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7440 anyhow::bail!(
7441 "the final commit moved during public description generation; retry before publishing"
7442 );
7443 }
7444 let note = match (&summarized, &self.state.change_summary) {
7445 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7446 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7447 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7448 (None, _) => ChangeNote::NotAsked,
7449 };
7450 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7451 if !posting {
7452 let identity = crate::scrub::Identity::current();
7453 return Ok(Some(PrMessage {
7454 title: crate::scrub::scrub(&pr.title, &identity),
7455 body: crate::scrub::scrub(&pr.body, &identity),
7456 }));
7457 }
7458 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7461 if let Some(known) = self
7462 .state
7463 .github_text
7464 .clone()
7465 .filter(|g| g.fingerprint == raw_fp)
7466 {
7467 return self.settle_withheld(winner, &pr, known).await;
7468 }
7469 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7470 let mut violations =
7471 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7472 let identity = crate::scrub::Identity::current();
7473 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7474 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7475 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7476 {
7477 violations.push(crate::github_text::Violation::SensitiveData);
7478 }
7479 let has_language = violations
7483 .iter()
7484 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7485 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7486 self.state.event(
7487 "github-text",
7488 if has_language {
7489 format!("description rejected: {violations:?}; requesting one rewrite")
7490 } else {
7491 format!("description rejected: {violations:?}; asking the owner")
7492 },
7493 );
7494 let mut rewritten = false;
7495 let mut rejected_rewrite = None;
7496 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7497 {
7498 let output = text;
7499 if let Some(title) = summary_title(&output) {
7500 let raw_body = summary_without_title(&output);
7503 let clean_title = crate::scrub::scrub(&title, &identity);
7504 let mut body = crate::scrub::scrub(&raw_body, &identity);
7505 let redacted = clean_title != title || body != raw_body;
7506 let title = clean_title;
7507 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7508 if !complete {
7509 self.state.event(
7510 "github-text",
7511 "description rewrite omitted required findings",
7512 );
7513 }
7514 if !body.trim().is_empty() {
7515 body.push_str("\n\n");
7516 body.push_str(&public_pr_evidence(&self.state, head));
7517 }
7518 let marker = format!("magi:run/{}", self.state.id);
7519 if !body.trim().is_empty() && !body.contains(&marker) {
7520 body.push_str(&format!("\n\n{marker}"));
7521 }
7522 let rewrite = PrMessage {
7523 title: title.clone(),
7524 body: body.clone(),
7525 };
7526 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7527 let mut failed = crate::github_text::check_for(
7528 &self.state.config.graph,
7529 &title,
7530 &body,
7531 redecision,
7532 );
7533 if (!crate::github_text::shareable(&title)
7534 || !crate::github_text::shareable(&body))
7535 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7536 {
7537 failed.push(crate::github_text::Violation::SensitiveData);
7538 }
7539 if complete
7540 && !title.trim().is_empty()
7541 && !body.trim().is_empty()
7542 && !failed.is_empty()
7543 {
7544 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7545 title: title.clone(),
7546 body: body.clone(),
7547 failed: failed
7548 .iter()
7549 .map(|v| crate::github_text::category(*v).to_owned())
7550 .collect(),
7551 });
7552 }
7553 if complete
7554 && !title.trim().is_empty()
7555 && !raw_body.trim().is_empty()
7556 && failed.is_empty()
7557 {
7558 decision = redecision;
7559 pr = PrMessage { title, body };
7560 rewritten = true;
7561 self.state
7562 .event("github-text", "description rewrite passed");
7563 if redacted {
7564 self.state.event(
7565 "github-text",
7566 "sensitive data removed from description rewrite",
7567 );
7568 }
7569 }
7570 }
7571 }
7572 if !rewritten {
7573 self.state.event(
7574 "github-text",
7575 "description rewrite unavailable or rejected; asking the owner",
7576 );
7577 if let Some(w) =
7578 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7579 {
7580 self.retire_github_text_question("the withheld text changed");
7583 let record = crate::run::GithubTextAsk {
7584 rewrite: rejected_rewrite,
7585 fingerprint: raw_fp,
7586 title: w.title,
7587 body: w.body,
7588 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7589 question: None,
7590 asks: 0,
7591 resolved: false,
7592 chosen_title: None,
7593 choice: None,
7594 };
7595 return self.settle_withheld(winner, &pr, record).await;
7596 }
7597 }
7598 }
7599 let (title, body) =
7600 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7601 Ok(Some(PrMessage { title, body }))
7602 }
7603
7604 async fn summarize_change(
7615 &mut self,
7616 winner: &Candidate,
7617 head: Option<&str>,
7618 ) -> std::result::Result<(), String> {
7619 let Some(head) = head else {
7620 return Err("the final commit could not be read".to_owned());
7621 };
7622 let remote = self.state.config.merge.remote.clone();
7623 let tracking = format!("{remote}/{}", self.state.base_branch);
7624 if !git::rev_exists(&winner.worktree, &tracking).await {
7625 return Err(format!("the remote base `{tracking}` could not be read"));
7626 }
7627 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7628 Ok(b) if !b.is_empty() => b,
7629 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7630 };
7631 let evidence = crate::pr_outcome::summary_evidence(
7632 &self.state,
7633 crate::pr_outcome::FinalHead::At(head),
7634 );
7635 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7636 if self
7637 .state
7638 .change_summary
7639 .as_ref()
7640 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7641 {
7642 return Ok(());
7643 }
7644 if self.state.change_summary.take().is_some() {
7645 self.state.event(
7646 "pr-summary",
7647 "the final diff or evidence changed; the earlier change summary was discarded",
7648 );
7649 }
7650 if let Err(e) = write_artifact(
7652 &self.state,
7653 &format!("public-description-{fingerprint}.md"),
7654 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7655 ) {
7656 tracing::warn!("could not save public description evidence: {e:#}");
7657 }
7658 let chain = match agent::pick_chain(
7659 &self.state.config.agents,
7660 self.state.config.roles.synthesizer.as_ref(),
7661 &agent::installed,
7662 "synthesizer",
7663 ) {
7664 Ok(chain) => chain,
7665 Err(e) => {
7666 tracing::warn!("no agent for the change summary: {e:#}");
7667 self.state.event("pr-summary", "no summary agent available");
7668 return Err("no agent was available to summarize it".to_owned());
7669 }
7670 };
7671 let subjects = git::subjects(&winner.worktree, &base, head)
7672 .await
7673 .unwrap_or_default();
7674 let stat = git::diff_stat(&winner.worktree, &base, head)
7675 .await
7676 .unwrap_or_default();
7677 let stat: String = stat.chars().take(6_000).collect();
7678 let mut prompt = crate::prompt::change_summary_in(
7679 &base,
7680 head,
7681 &subjects,
7682 stat.trim(),
7683 &self.state.config.graph.github_language,
7684 );
7685 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7686 let sessions = self.state.config.graph.sessions;
7687 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7688 let prompts = self.state.config.prompts.clone();
7689 let cache = self.state.config.cache_dir();
7690 let run = self.state.id.clone();
7691 for (i, spec) in chain.into_iter().enumerate() {
7692 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7693 let github_spec = match agent::for_github_text(
7694 &spec,
7695 &self.state.config.graph.github_language,
7696 &winner.worktree,
7697 ) {
7698 Ok(spec) => spec,
7699 Err(e) => {
7700 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7701 continue;
7702 }
7703 };
7704 let job = SeatJob {
7705 spec: github_spec,
7706 seat,
7707 cwd: winner.worktree.clone(),
7708 prompt: prompt.clone(),
7709 timeout,
7710 allow_write: false,
7711 sessions,
7712 artifacts: agent::artifacts_dir(&self.state.dir()),
7713 stem: if i == 0 {
7714 "change-summary".to_owned()
7715 } else {
7716 format!("change-summary-{}", spec.id)
7717 },
7718 handover: None,
7719 };
7720 let ctx = WaveCtx {
7721 run: &run,
7722 node: "pr-summary",
7723 prompts: &prompts,
7724 cache: cache.as_deref(),
7725 round: None,
7726 carry_seats: false,
7727 pauses: Some((&self.pause, &self.interrupt)),
7728 };
7729 let (seat, outcome) =
7730 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7731 self.state.seats.insert(seat.key.clone(), seat);
7732 let text = match outcome {
7733 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7734 AgentOutcome::ResourceBlocked(why) => {
7735 if self.state.resource_block.is_none() {
7738 self.state.resource_block = Some(crate::run::ResourceBlock {
7739 node: "pr-summary".to_owned(),
7740 kind: "shared resource".to_owned(),
7741 detail: why,
7742 at: Timestamp::now(),
7743 });
7744 }
7745 return Err("a shared resource was busy".to_owned());
7746 }
7747 _ => continue,
7748 };
7749 let blind = &self.state.config.blind;
7750 let Some(title) = summary_title(&text) else {
7751 continue;
7752 };
7753 let body: String = summary_without_title(&text)
7754 .lines()
7755 .filter(|l| !l.contains("magi:run/"))
7756 .collect::<Vec<_>>()
7757 .join("\n");
7758 let body = blind::sanitize_prose(body.trim(), blind);
7759 let title = blind::sanitize_prose(&title, blind);
7760 if body.trim().is_empty() || title.trim().is_empty() {
7761 continue;
7762 }
7763 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7764 continue;
7765 }
7766 if git::rev_parse(&winner.worktree, "HEAD")
7768 .await
7769 .ok()
7770 .as_deref()
7771 != Some(head)
7772 {
7773 return Err("the branch moved while the summary was written".to_owned());
7774 }
7775 if git::merge_base(&winner.worktree, &tracking, head)
7776 .await
7777 .ok()
7778 .as_deref()
7779 != Some(base.as_str())
7780 || crate::pr_outcome::summary_evidence(
7781 &self.state,
7782 crate::pr_outcome::FinalHead::At(head),
7783 ) != evidence
7784 {
7785 return Err("the summary evidence changed while it was written".to_owned());
7786 }
7787 self.state.change_summary = Some(crate::run::ChangeSummary {
7788 head: head.to_owned(),
7789 base,
7790 evidence: fingerprint,
7791 title,
7792 body,
7793 });
7794 self.state.event(
7795 "pr-summary",
7796 format!("change summary written by {} for the final diff", spec.id),
7797 );
7798 if let Err(e) = self.state.save() {
7799 tracing::warn!("could not save the change summary: {e:#}");
7800 }
7801 return Ok(());
7802 }
7803 self.state
7804 .event("pr-summary", "no agent produced a change summary");
7805 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7806 }
7807
7808 async fn rewrite_pr_text(
7818 &mut self,
7819 winner: &Candidate,
7820 pr: &PrMessage,
7821 violations: &[crate::github_text::Violation],
7822 ) -> Option<String> {
7823 let prompt = format!(
7824 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7825 pr.title, pr.body
7826 );
7827 let sessions = self.state.config.graph.sessions;
7828 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7829 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7832 let key = format!("impl-{}", winner.label);
7833 if let Some(spec) = self
7834 .state
7835 .config
7836 .agents
7837 .iter()
7838 .find(|a| a.id == winner.agent)
7839 .cloned()
7840 && let Some(existing) = self.state.seats.get(&key)
7841 && existing.agent == spec.id
7842 && agent::has_session(spec.kind, existing, sessions)
7843 {
7844 attempts.push((spec, Some(existing.clone())));
7845 }
7846 match agent::pick_chain(
7847 &self.state.config.agents,
7848 self.state.config.roles.language_judge.as_ref(),
7849 &agent::installed,
7850 "language_judge",
7851 ) {
7852 Ok(chain) => {
7853 for spec in chain {
7854 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7855 continue;
7856 }
7857 attempts.push((spec, None));
7858 }
7859 }
7860 Err(e) if attempts.is_empty() => {
7861 tracing::warn!("no agent for the description rewrite: {e:#}");
7862 self.state
7863 .event("github-text", "no rewrite agent available");
7864 return None;
7865 }
7866 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7867 }
7868 let prompts = self.state.config.prompts.clone();
7869 let cache = self.state.config.cache_dir();
7870 let run = self.state.id.clone();
7871 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7872 let resumed = resumed_seat.is_some();
7873 let seat = resumed_seat.unwrap_or_else(|| {
7874 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7875 });
7876 let github_spec = match agent::for_github_text(
7877 &spec,
7878 &self.state.config.graph.github_language,
7879 &winner.worktree,
7880 ) {
7881 Ok(spec) => spec,
7882 Err(e) => {
7883 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7884 continue;
7885 }
7886 };
7887 let job = SeatJob {
7888 spec: github_spec,
7889 seat,
7890 cwd: winner.worktree.clone(),
7891 prompt: prompt.clone(),
7892 timeout: retry_budget(timeout_full, resumed),
7893 allow_write: false,
7894 sessions,
7895 artifacts: agent::artifacts_dir(&self.state.dir()),
7896 stem: if i == 0 {
7897 "github-text-rewrite".to_owned()
7898 } else {
7899 format!("github-text-rewrite-{}", spec.id)
7900 },
7901 handover: None,
7902 };
7903 let ctx = WaveCtx {
7904 run: &run,
7905 node: "github-text",
7906 prompts: &prompts,
7907 cache: cache.as_deref(),
7908 round: None,
7909 carry_seats: false,
7910 pauses: Some((&self.pause, &self.interrupt)),
7911 };
7912 let (seat, outcome) =
7913 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7914 self.state.seats.insert(seat.key.clone(), seat);
7915 match outcome {
7916 AgentOutcome::Ok(o)
7917 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7918 {
7919 return Some(o.text);
7920 }
7921 AgentOutcome::ResourceBlocked(_) => return None,
7922 _ => {}
7923 }
7924 }
7925 None
7926 }
7927
7928 fn retire_github_text_question(&mut self, why: &str) {
7930 let Some(id) = self
7931 .state
7932 .github_text
7933 .as_ref()
7934 .and_then(|g| g.question.clone())
7935 else {
7936 return;
7937 };
7938 let _ = ask::Questions::open().update(&id, |q| {
7939 q.abandon(why);
7940 Ok(())
7941 });
7942 }
7943
7944 async fn settle_withheld(
7948 &mut self,
7949 winner: &Candidate,
7950 pr: &PrMessage,
7951 mut g: crate::run::GithubTextAsk,
7952 ) -> Result<Option<PrMessage>> {
7953 use crate::github_text as gt;
7954 let store = ask::Questions::open();
7955 let timeout = self.state.config.graph.answer_timeout;
7956 if !g.resolved {
7957 let mut standing = g
7958 .question
7959 .as_ref()
7960 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7961 if let Some(q) = standing.as_ref()
7962 && q.status.open()
7963 && gt::expired(q, timeout)
7964 {
7965 let why = format!("no answer within {}s of asking", timeout.max(1));
7966 standing = store
7967 .update(&q.id, |q| {
7968 q.abandon(&why);
7969 Ok(())
7970 })
7971 .ok()
7972 .map(|(q, ())| q);
7973 }
7974 let mut ask_again: Option<Option<Vec<String>>> = None;
7975 match standing {
7976 None if g.asks <= 1 => {
7979 g.asks = 0;
7980 ask_again = Some(None);
7981 }
7982 None => {
7983 g.resolved = true;
7984 g.chosen_title = None;
7985 g.choice = Some(GithubTextChoice::Fallback);
7986 }
7987 Some(q) if q.status.open() => {
7988 self.park_on_github_text(&g)?;
7989 return Ok(None);
7990 }
7991 Some(q) => match gt::read_reply(&q) {
7992 gt::Reply::Fallback => {
7993 g.resolved = true;
7994 g.chosen_title = None;
7995 g.choice = Some(GithubTextChoice::Fallback);
7996 }
7997 gt::Reply::Original => {
7998 g.resolved = true;
7999 g.chosen_title = None;
8000 g.choice = Some(GithubTextChoice::Original);
8001 }
8002 gt::Reply::Rewrite => {
8003 g.resolved = true;
8004 g.chosen_title = None;
8005 g.choice = Some(if g.rewrite.is_some() {
8006 GithubTextChoice::Rewrite
8007 } else {
8008 GithubTextChoice::Fallback
8009 });
8010 }
8011 gt::Reply::Title(text) => {
8012 let first = text
8013 .lines()
8014 .map(str::trim)
8015 .find(|l| !l.is_empty())
8016 .unwrap_or("");
8017 let decision = if first.is_empty() {
8018 None
8019 } else {
8020 let candidate = PrMessage {
8021 title: first.to_owned(),
8022 body: String::new(),
8023 };
8024 self.judge_pr_language(&winner.worktree, &candidate).await
8025 };
8026 match gt::vet_title(
8027 &text,
8028 decision,
8029 self.state.config.graph.github_english(),
8030 ) {
8031 Ok(title) => {
8032 g.resolved = true;
8033 g.chosen_title = Some(title);
8034 g.choice = Some(GithubTextChoice::MyTitle);
8035 }
8036 Err(cats) if g.asks < 2 => {
8037 ask_again = Some(Some(
8038 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8039 ))
8040 }
8041 Err(_) => {
8042 g.resolved = true;
8043 g.chosen_title = None;
8044 g.choice = Some(GithubTextChoice::Fallback);
8045 }
8046 }
8047 }
8048 },
8049 }
8050 if let Some(rejected) = ask_again {
8051 let w = gt::Withheld {
8054 title: g.title,
8055 body: g.body,
8056 categories: g.categories.clone(),
8057 };
8058 let artifact = match crate::run::write_artifact(
8061 &self.state,
8062 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8063 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8064 ) {
8065 Ok(p) => Some(p),
8066 Err(e) => {
8067 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8068 None
8069 }
8070 };
8071 let (detail, comparison) = gt::question_content(
8072 &w,
8073 &pr.title,
8074 &pr.body,
8075 &self.state.id,
8076 rejected.as_deref(),
8077 artifact.as_deref(),
8078 g.rewrite.as_ref(),
8079 );
8080 let mut q = ask::Question::new(
8081 self.state.id.clone(),
8082 gt::ASK_NODE.to_owned(),
8083 gt::ASK_SEAT.to_owned(),
8084 gt::question_summary(&self.state.config.graph.language, &w),
8085 detail,
8086 gt::question_choices(&w, g.rewrite.as_ref()),
8087 );
8088 q.posting_comparison = Some(comparison);
8089 q.answer_timeout = timeout;
8090 g.question = Some(q.id.clone());
8093 g.asks += 1;
8094 self.state.github_text = Some(g.clone());
8095 self.state.event(
8096 "github-text",
8097 format!("asking the owner about the withheld text ({})", q.short()),
8098 );
8099 self.state.save()?;
8100 store
8101 .put_executable(&mut q)
8102 .context("file the posting-gate question")?;
8103 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8104 tracing::warn!(
8105 "could not notify about posting-gate question {}: {e:#}",
8106 q.short()
8107 );
8108 }
8109 self.park_on_github_text(&g)?;
8110 return Ok(None);
8111 }
8112 }
8113 let resolved = g.chosen_title.clone();
8114 let original = g.choice == Some(GithubTextChoice::Original);
8115 self.state.event(
8117 "github-text",
8118 if original {
8119 format!(
8120 "posting the original text for: {}",
8121 match (g.title, g.body) {
8122 (true, true) => "title, description",
8123 (true, false) => "title",
8124 _ => "description",
8125 }
8126 )
8127 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8128 "posting the saved rewrite title and description".to_owned()
8129 } else if resolved.is_some() {
8130 "posting the owner's replacement title".to_owned()
8131 } else {
8132 "posting the neutral text for the withheld fields".to_owned()
8133 },
8134 );
8135 self.state.github_text = Some(g.clone());
8136 self.state.save()?;
8137 if g.choice == Some(GithubTextChoice::Rewrite)
8138 && let Some(r) = g.rewrite.as_ref()
8139 {
8140 return Ok(Some(PrMessage {
8141 title: r.title.clone(),
8142 body: r.body.clone(),
8143 }));
8144 }
8145 let w = gt::Withheld {
8146 title: g.title,
8147 body: g.body,
8148 categories: g.categories.clone(),
8149 };
8150 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8151 if let Some(t) = resolved {
8152 title = t;
8153 }
8154 let english = gt::LanguageDecision {
8156 title_english: true,
8157 body_english: true,
8158 body_complete: true,
8159 };
8160 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8161 if original {
8162 if g.title {
8165 title = pr.title.clone();
8166 }
8167 if g.body {
8168 body = pr.body.clone();
8169 }
8170 }
8171 Ok(Some(PrMessage { title, body }))
8172 }
8173
8174 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8175 self.state.github_text = Some(g.clone());
8176 self.state.end_pass_parked(ParkCause::Question);
8177 self.state.parked = true;
8178 self.state.event(
8179 "github-text",
8180 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8181 );
8182 self.state.save()
8183 }
8184
8185 async fn judge_pr_language(
8188 &mut self,
8189 cwd: &Path,
8190 pr: &PrMessage,
8191 ) -> Option<crate::github_text::LanguageDecision> {
8192 self.state.config.roles.language_judge.as_ref()?;
8193 if !self.state.config.graph.language_gate() {
8194 return None;
8195 }
8196 let decision =
8197 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8198 self.state.event(
8199 "github-text",
8200 if decision.is_some() {
8201 "language decided by the language judge"
8202 } else {
8203 "language judge unavailable; using built-in heuristics"
8204 },
8205 );
8206 decision
8207 }
8208
8209 async fn merge(&mut self) -> Result<()> {
8210 if self
8225 .state
8226 .base_sync
8227 .as_ref()
8228 .is_some_and(|s| s.conflict.is_some())
8229 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8230 != Some(RunStatus::Gating)
8231 || !self.state.gate_allows_progress()
8239 {
8240 return Ok(());
8241 }
8242 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8251 return Ok(());
8252 }
8253 let Some(winner) = self.state.winner().cloned() else {
8254 return Ok(());
8255 };
8256 let repo = self.state.repo.clone();
8257 let base = self.state.base_branch.clone();
8258 let mode = self.state.config.merge.mode;
8259 let style = self.state.config.merge.style;
8260 let facts = if is_review_run(&self.state) {
8261 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8262 let start = review_base(
8263 &repo,
8264 &self.state.config.merge.remote,
8265 &base,
8266 &self.state.base_commit,
8267 &winner.branch,
8268 )
8269 .await;
8270 branch_facts(&repo, &start, &winner.branch).await
8271 } else {
8272 None
8273 };
8274 let leaked = if is_review_run(&self.state) {
8277 leaked_subjects(&self.state, &winner.branch).await
8278 } else {
8279 Some(Vec::new())
8280 };
8281 let Some(pr) = self
8285 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8286 .await?
8287 else {
8288 return Ok(());
8289 };
8290 let message = pr.commit_message();
8291
8292 let outcome = match mode {
8293 MergeMode::None => MergeOutcome {
8294 mode,
8295 ok: true,
8296 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8297 empty: false,
8298 },
8299 MergeMode::Pr | MergeMode::Local
8300 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8301 {
8302 MergeOutcome {
8303 mode,
8304 ok: false,
8305 detail: empty_candidate_detail(&self.state, &base),
8306 empty: true,
8307 }
8308 }
8309 MergeMode::Local => {
8310 let on = git::current_branch(&repo).await?;
8311 if on.as_deref() != Some(base.as_str()) {
8312 MergeOutcome {
8313 mode,
8314 ok: false,
8315 detail: format!(
8316 "{} has {} checked out, not the base branch {base}",
8317 repo.display(),
8318 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8319 ),
8320 empty: false,
8321 }
8322 } else if !git::is_clean(&repo).await? {
8323 MergeOutcome {
8324 mode,
8325 ok: false,
8326 detail: format!("{} is dirty; refusing to merge", repo.display()),
8327 empty: false,
8328 }
8329 } else {
8330 let out = match style {
8331 MergeStyle::Merge => {
8332 git::merge_no_ff(&repo, &winner.branch, &message).await?
8333 }
8334 MergeStyle::Squash => {
8335 git::merge_squash(&repo, &winner.branch, &message).await?
8336 }
8337 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8338 };
8339 MergeOutcome {
8340 mode,
8341 ok: out.ok(),
8342 detail: if out.ok() { out.stdout } else { out.stderr },
8343 empty: false,
8344 }
8345 }
8346 }
8347 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8348 self.state.pr_publication_pending = true;
8349 self.state.save()?;
8350 let out =
8351 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8352 self.state.pr_publication_pending = out.is_err();
8353 MergeOutcome {
8354 mode,
8355 ok: out.is_ok(),
8356 detail: match out {
8357 Ok(url) => url,
8358 Err(e) => format!("{e:#}"),
8359 },
8360 empty: false,
8361 }
8362 }
8363 MergeMode::Pr => {
8364 let remote = self.state.config.merge.remote.clone();
8365 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8366 if !pushed.ok() {
8367 MergeOutcome {
8368 mode,
8369 ok: false,
8370 detail: pushed.stderr,
8371 empty: false,
8372 }
8373 } else {
8374 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8380 let out = match pr_merge_plan(found) {
8381 PrPlan::Create => {
8382 gh_pr_create(
8383 &winner.worktree,
8384 &base,
8385 &winner.branch,
8386 &pr.title,
8387 &pr.body,
8388 )
8389 .await
8390 }
8391 PrPlan::Adopt { url, title } => {
8392 self.state
8393 .event("merge", format!("Pr: adopted open pull request {url}"));
8394 if self.state.github_text.as_ref().is_some_and(|g| {
8395 g.resolved
8396 && g.choice == Some(GithubTextChoice::Rewrite)
8397 && g.rewrite.is_some()
8398 }) {
8399 land::set_pr_text_verbatim(
8402 &winner.worktree,
8403 &url,
8404 &pr.title,
8405 &pr.body,
8406 )
8407 .await
8408 .map(|()| url)
8409 } else {
8410 if title != pr.title
8411 && (!is_review_run(&self.state)
8412 || leaked
8413 .as_deref()
8414 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8415 && let Err(e) =
8416 if self.state.github_text.as_ref().is_some_and(|g| {
8417 g.resolved
8418 && g.title
8419 && g.choice == Some(GithubTextChoice::Original)
8420 }) {
8421 land::set_pr_title_verbatim(
8423 &winner.worktree,
8424 &url,
8425 &pr.title,
8426 )
8427 .await
8428 } else {
8429 land::set_pr_title(
8430 &mut self.state,
8431 &winner.worktree,
8432 &url,
8433 &pr.title,
8434 )
8435 .await
8436 }
8437 {
8438 tracing::warn!("could not refresh title of {url}: {e:#}");
8439 self.state
8440 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8441 }
8442 Ok(url)
8443 }
8444 }
8445 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8446 };
8447 match out {
8448 Ok(url) => MergeOutcome {
8449 mode,
8450 ok: true,
8451 detail: url,
8452 empty: false,
8453 },
8454 Err(e) => MergeOutcome {
8455 mode,
8456 ok: false,
8457 detail: e.to_string(),
8458 empty: false,
8459 },
8460 }
8461 }
8462 }
8463 };
8464
8465 self.state.status = match (mode, outcome.ok) {
8466 (MergeMode::None, _) => RunStatus::Ready,
8467 (_, true) if outcome.empty => RunStatus::Ready,
8468 (MergeMode::Pr, true) => RunStatus::Ready,
8469 (_, true) => RunStatus::Merged,
8470 (_, false) => RunStatus::Blocked,
8471 };
8472 self.state.event(
8473 "merge",
8474 format!(
8475 "{:?}: {}",
8476 mode,
8477 outcome.detail.lines().next().unwrap_or("")
8478 ),
8479 );
8480 self.state.merge = Some(outcome);
8481 self.state.save()?;
8482
8483 if self.state.config.graph.land
8489 && mode == MergeMode::Pr
8490 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8491 {
8492 self.run_land().await?;
8493 }
8494 self.settle_questions();
8499 Ok(())
8500 }
8501
8502 async fn run_land(&mut self) -> Result<()> {
8513 crate::pr_head::ensure_owner(&self.state)?;
8514 let url = self
8515 .state
8516 .pr
8517 .as_ref()
8518 .map(|pr| pr.url.clone())
8519 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8520 .unwrap_or_default();
8521 let url = url.lines().next().unwrap_or("").trim().to_owned();
8522 if !url.starts_with("http") {
8523 return Ok(());
8524 }
8525 loop {
8528 match land::land(&mut self.state, &url).await {
8529 Ok(_)
8530 if self
8531 .state
8532 .merge_repairs
8533 .last()
8534 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8535 {
8536 if self.repair_merge_review().await? {
8537 continue;
8538 }
8539 }
8540 Ok(pr) if self.state.parked => {
8541 let _ = pr;
8545 }
8546 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8547 self.state.status = RunStatus::Superseded;
8548 self.state.save()?;
8549 }
8550 Ok(pr) => {
8551 self.state.status = match pr.state {
8552 land::PrLifecycle::Merged => RunStatus::Merged,
8553 _ => RunStatus::Blocked,
8554 };
8555 if bump::should_release_bump(self.state.status)
8562 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8563 {
8564 self.state
8570 .event("bump", format!("release bump skipped: {e:#}"));
8571 }
8572 if self.state.status == RunStatus::Merged {
8575 crate::followup::after_merge(&mut self.state, &pr.url).await;
8576 }
8577 self.state.save()?;
8578 }
8579 Err(e) => {
8580 if e.is::<crate::repo_lock::ResourceBlocked>() {
8581 return Err(e);
8582 }
8583 self.state.status = RunStatus::Blocked;
8584 self.state.event("land", format!("gave up: {e}"));
8585 self.state.save()?;
8586 }
8587 }
8588 break;
8589 }
8590 Ok(())
8591 }
8592
8593 async fn repair_merge_review(&mut self) -> Result<bool> {
8596 use crate::run::MergeRepairPhase as Phase;
8597 crate::pr_head::ensure_owner(&self.state)?;
8598 let i = self.state.merge_repairs.len() - 1;
8599 let request = self.state.merge_repairs[i].clone();
8600 if request.phase == Phase::Fixing {
8601 let restored = match self.state.winner().cloned() {
8602 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8603 None => Err(anyhow::anyhow!("repair has no winning branch")),
8604 };
8605 let why = match restored {
8606 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8607 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8608 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8609 };
8610 self.hold_merge_repair(i, why)?;
8611 return Ok(false);
8612 }
8613 let failure = if i >= self.state.config.graph.land_rounds
8614 || self.state.config.graph.review_rounds == 0
8615 {
8616 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8617 } else {
8618 None
8619 };
8620 if let Some(why) = failure {
8621 self.hold_merge_repair(i, why)?;
8622 return Ok(false);
8623 }
8624 if request.phase == Phase::Requested {
8625 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8626 Ok(pr) => pr,
8627 Err(e) => {
8628 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8629 return Ok(false);
8630 }
8631 };
8632 self.state.merge_repairs[i].phase = Phase::Fixing;
8633 self.state.land_approval = None;
8634 self.state.land_armed_head = None;
8635 self.state.save()?;
8636 let evidence = serde_json::to_string_pretty(&request)?;
8639 let reason = format!(
8640 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8641 );
8642 let budget = self.state.config.graph.land_rounds;
8643 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8644 match result {
8645 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8646 Ok(land::Fixed::Failed(why)) => {
8647 self.hold_merge_repair(
8648 i,
8649 format!(
8650 "repair failed: {why}; inspect the branch and run an explicit review"
8651 ),
8652 )?;
8653 return Ok(false);
8654 }
8655 Err(e) => {
8656 self.hold_merge_repair(
8657 i,
8658 format!(
8659 "repair failed: {e:#}; inspect the branch and run an explicit review"
8660 ),
8661 )?;
8662 return Ok(false);
8663 }
8664 }
8665 self.begin_merge_repair_review(i)?;
8669 }
8670 self.review_loop().await?;
8671 self.gate().await?;
8672 if self.resource_halt()? || self.state.parked {
8673 return Ok(false);
8674 }
8675 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8676 != Some(RunStatus::Gating)
8677 || !self.state.gate_allows_progress()
8678 {
8679 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8680 return Ok(false);
8681 }
8682 let winner = self
8685 .state
8686 .winner()
8687 .cloned()
8688 .context("repair has no winning branch")?;
8689 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8690 if self.state.gate_head.as_deref() != Some(after.as_str())
8691 || !git::is_clean(&winner.worktree).await?
8692 {
8693 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8694 return Ok(false);
8695 }
8696 let expected = self
8697 .state
8698 .pr_pushed_head
8699 .as_deref()
8700 .unwrap_or(&request.head)
8701 .to_owned();
8702 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8703 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8706 let refspec = format!("{after}:refs/heads/{public}");
8707 let push = git::git_raw(
8708 &winner.worktree,
8709 &["push", &lease, &self.state.config.merge.remote, &refspec],
8710 )
8711 .await;
8712 match push {
8713 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8714 Ok(out) => {
8715 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8716 return Ok(false);
8717 }
8718 Err(e) => {
8719 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8720 return Ok(false);
8721 }
8722 }
8723 self.state.merge_repairs[i].phase = Phase::Complete;
8724 self.state.status = RunStatus::Landing;
8725 self.state.event(
8726 "repair",
8727 "new review and gate completed; fresh owner merge decision required",
8728 );
8729 self.state.save()?;
8730 Ok(true)
8731 }
8732
8733 fn merge_repair_artifacts(&self) -> PathBuf {
8734 let artifacts = agent::artifacts_dir(&self.state.dir());
8735 match self
8736 .state
8737 .merge_repairs
8738 .last()
8739 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8740 {
8741 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8742 None => artifacts,
8743 }
8744 }
8745
8746 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8747 self.state.reviews.clear();
8748 self.state.gate.clear();
8749 self.state.gate_ran = false;
8750 self.state.gate_head = None;
8751 self.state.gate_fixes.clear();
8752 self.state.gate_repair_e2e = None;
8753 self.state.pre_gate.clear();
8754 self.state.pre_gate_commit = None;
8755 self.state.contested_handoff = None;
8756 self.state.land_reruns = None;
8757 self.state.land_approval = None;
8758 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8759 self.state.status = RunStatus::Reviewing;
8760 self.state.save()
8761 }
8762
8763 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8764 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8765 self.state.merge_repairs[index].failure = Some(why.clone());
8766 self.state.status = RunStatus::Blocked;
8767 self.state.event("repair", why);
8768 self.state.save()
8769 }
8770
8771 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8775 if let Some(existing) = self.state.seats.get(key)
8776 && existing.agent == agent
8777 {
8778 return existing.clone();
8779 }
8780 let fresh = if self.state.seats.contains_key(key) {
8783 handover_seat(key, agent, self.state.next_seat_seed())
8784 } else {
8785 SeatState::new(key, agent, self.state.seed)
8786 };
8787 self.state.seats.insert(key.to_owned(), fresh.clone());
8788 fresh
8789 }
8790
8791 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8797 match self.state.seats.get(key) {
8798 Some(s) if s.agent != spec.id => {
8799 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8800 }
8801 _ => spec,
8802 }
8803 }
8804
8805 fn view(&self, c: &Candidate) -> CandidateView {
8807 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8808 .unwrap_or_default();
8809 let (patch, _) = blind::sanitize_patch(
8810 &format!("candidate {} patch", c.label),
8811 &raw,
8812 &self.state.config.blind,
8813 );
8814 CandidateView {
8815 label: c.label,
8816 branch: c.branch.clone(),
8817 summary: c.summary.clone(),
8818 stat: c.stat.clone(),
8819 patch,
8820 }
8821 }
8822
8823 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8825 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8826 prompt::judge(
8827 "(see above)",
8828 &views,
8829 self.roles.judges.len(),
8830 base_short,
8831 "en",
8832 )
8833 }
8834
8835 fn vote_prompt_full(
8841 &self,
8842 j: usize,
8843 labels: &[char],
8844 language: &str,
8845 candidates: &[Candidate],
8846 base_short: &str,
8847 ) -> String {
8848 let mut text = format!(
8849 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8850 prompt::final_vote(labels, language),
8851 self.state.instruction,
8852 self.candidate_block(candidates, base_short)
8853 );
8854 if let Some(own) = self
8855 .state
8856 .judgements
8857 .get(j)
8858 .filter(|r| !r.ranking.is_empty())
8859 {
8860 let reasons = own
8861 .reasons
8862 .iter()
8863 .map(|(k, v)| format!("- {k}: {v}"))
8864 .collect::<Vec<_>>()
8865 .join("\n");
8866 text.push_str(&format!(
8867 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8868 own.ranking.iter().collect::<String>(),
8869 if reasons.is_empty() {
8870 ""
8871 } else {
8872 ", because:\n"
8873 }
8874 ));
8875 }
8876 if !self.state.deliberation.is_empty() {
8877 text.push_str("\n# What was argued before this vote\n");
8878 for t in self.transcript(&[], j) {
8879 text.push_str(&format!(
8880 "\n## {}{}\n\n{}\n",
8881 t.who,
8882 if t.is_self { " (you)" } else { "" },
8883 t.body.trim()
8884 ));
8885 }
8886 }
8887 text
8888 }
8889
8890 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8897 let mut turns = Vec::new();
8898 for j in &self.state.judgements {
8899 if j.ranking.is_empty() {
8900 continue;
8901 }
8902 let reasons = j
8903 .reasons
8904 .iter()
8905 .map(|(k, v)| format!("- {k}: {v}"))
8906 .collect::<Vec<_>>()
8907 .join("\n");
8908 turns.push(Turn {
8909 who: format!("Judge {} (opening ranking)", j.judge),
8910 is_self: j.judge == self_idx + 1,
8911 body: format!(
8912 "Ranked {}{}{reasons}",
8913 j.ranking.iter().collect::<String>(),
8914 if reasons.is_empty() {
8915 ""
8916 } else {
8917 ", because:\n"
8918 }
8919 ),
8920 });
8921 }
8922 for t in self
8923 .state
8924 .deliberation
8925 .iter()
8926 .flat_map(|r| r.turns.iter())
8927 .chain(current)
8928 {
8929 turns.push(Turn {
8930 who: format!("Judge {}", t.judge),
8931 is_self: t.judge == self_idx + 1,
8932 body: t.body.clone(),
8933 });
8934 }
8935 turns
8936 }
8937}
8938
8939fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8941 agent::has_session(spec.kind, seat, sessions)
8942}
8943
8944fn next_untried_in_roster<'a>(
8965 roster: &'a [AgentSpec],
8966 start: usize,
8967 tried: &BTreeSet<String>,
8968) -> Option<&'a AgentSpec> {
8969 roster
8970 .get(start + 1..)?
8971 .iter()
8972 .find(|s| !tried.contains(&s.id))
8973}
8974
8975fn pick_successor<'a>(
8996 roster: &'a [AgentSpec],
8997 start: usize,
8998 tried: &BTreeSet<String>,
8999 carried: Option<&BTreeSet<String>>,
9000 others: &BTreeSet<String>,
9001) -> Option<&'a AgentSpec> {
9002 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9003 roster
9004 .get(start + 1..)
9005 .and_then(|tail| {
9006 tail.iter()
9007 .filter(free)
9008 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9009 })
9010 .or_else(|| {
9011 carried
9012 .is_some()
9013 .then(|| roster.iter().find(free))
9014 .flatten()
9015 })
9016 .or_else(|| match carried {
9017 Some(c) => next_for_seat(roster, start, tried, c),
9018 None => next_untried_in_roster(roster, start, tried),
9019 })
9020}
9021
9022fn next_for_seat<'a>(
9032 roster: &'a [AgentSpec],
9033 start: usize,
9034 round_tried: &BTreeSet<String>,
9035 carried_failed: &BTreeSet<String>,
9036) -> Option<&'a AgentSpec> {
9037 roster
9038 .get(start + 1..)?
9039 .iter()
9040 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9041 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9042}
9043
9044fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9050 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9051 return spec;
9052 };
9053 let ok = |id: &str| !h.failed.contains(id);
9054 if let Some(last) = h.last_ok.as_deref()
9055 && ok(last)
9056 && let Some(s) = roster.iter().find(|s| s.id == last)
9057 {
9058 return s.clone();
9059 }
9060 if ok(&spec.id) {
9061 return spec;
9062 }
9063 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9064 roster
9065 .get(from + 1..)
9066 .into_iter()
9067 .flatten()
9068 .chain(roster.iter())
9069 .find(|s| ok(&s.id))
9070 .cloned()
9071 .unwrap_or(spec)
9072}
9073
9074pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9078 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9079}
9080
9081const TIMED_OUT: &str = "timed out";
9084
9085impl FailClass {
9086 fn of(out: &AgentOutcome) -> Option<Self> {
9088 match out {
9089 AgentOutcome::Ok(_) => None,
9090 AgentOutcome::Quota(_) => Some(Self::Quota),
9091 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9092 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9093 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9094 AgentOutcome::ResourceBlocked(_) => None,
9097 }
9098 }
9099
9100 fn stem_word(&self) -> &'static str {
9102 match self {
9103 Self::Quota => "quota",
9104 _ => "handover",
9105 }
9106 }
9107}
9108
9109fn failure_signature(msg: &str) -> String {
9113 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9114 if let Some(code) = line
9117 .strip_prefix("process exited with no usable output (exit code ")
9118 .and_then(|s| s.strip_suffix(')'))
9119 .and_then(|s| s.parse::<i32>().ok())
9120 {
9121 line = format!("exited with some({code}) and no usable output");
9122 } else if matches!(
9123 line.as_str(),
9124 "process exited with no usable output (no exit code reported)"
9125 | "process exited with no usable output (exit code unavailable)"
9126 ) {
9127 line = "exited with none and no usable output".to_owned();
9128 }
9129 let mut out = Vec::new();
9130 for word in line.split_whitespace() {
9131 if word.contains('/') || word.contains('\\') {
9132 out.push("<path>".to_owned());
9133 continue;
9134 }
9135 let mut w = String::new();
9136 let mut in_digits = false;
9137 for c in word.chars() {
9138 if c.is_ascii_digit() {
9139 if !in_digits {
9140 w.push('#');
9141 }
9142 in_digits = true;
9143 } else {
9144 in_digits = false;
9145 w.push(c);
9146 }
9147 }
9148 out.push(w);
9149 }
9150 out.join(" ").chars().take(120).collect()
9151}
9152
9153fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9160 match cur {
9161 FailClass::Quota | FailClass::Timeout => true,
9162 FailClass::Other(_) => prev != Some(cur),
9163 }
9164}
9165
9166fn fail_reason(out: &AgentOutcome) -> String {
9168 match out {
9169 AgentOutcome::Ok(_) => String::new(),
9170 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9171 AgentOutcome::Dropped(o) => format!(
9172 "the CLI dropped the stream ({})",
9173 o.dropped
9174 .as_ref()
9175 .map(|d| d.why.as_str())
9176 .unwrap_or("it ended without delivering its answer")
9177 ),
9178 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9179 e.lines().next().unwrap_or("").chars().take(160).collect()
9180 }
9181 }
9182}
9183
9184pub(crate) fn record_handover(
9187 state: &mut RunState,
9188 node: &str,
9189 seat: &str,
9190 from: &str,
9191 to: &str,
9192 class: &FailClass,
9193 reason: &str,
9194) {
9195 let message = if *class == FailClass::Quota {
9196 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9197 } else {
9198 format!("{seat}: handed over {from} -> {to} ({reason})")
9199 };
9200 state.event(node, message);
9201 state.handovers.push(Handover {
9202 at: Timestamp::now(),
9203 node: node.to_owned(),
9204 seat: seat.to_owned(),
9205 from: from.to_owned(),
9206 to: to.to_owned(),
9207 reason: reason.to_owned(),
9208 });
9209}
9210
9211fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9225 commands.iter().any(|c| c.exit_code.is_none())
9226}
9227
9228fn verified_noop_claim(
9241 usable: bool,
9242 commands: &[agent::CommandEvidence],
9243 text: &str,
9244) -> Option<String> {
9245 (usable && !has_unconfirmed_command(commands))
9246 .then(|| verdict::verified_noop(text))
9247 .flatten()
9248}
9249
9250fn short(commit: &str) -> String {
9251 commit.chars().take(7).collect()
9252}
9253
9254fn make_executable(path: &Path) -> Result<()> {
9255 #[cfg(unix)]
9256 {
9257 use std::os::unix::fs::PermissionsExt as _;
9258 let mut perms = std::fs::metadata(path)?.permissions();
9259 perms.set_mode(0o755);
9260 std::fs::set_permissions(path, perms)?;
9261 }
9262 #[cfg(not(unix))]
9263 {
9264 let _ = path;
9265 }
9266 Ok(())
9267}
9268
9269struct WaveCtx<'a> {
9276 run: &'a str,
9279 node: &'a str,
9281 prompts: &'a Prompts,
9282 cache: Option<&'a Path>,
9284 round: Option<usize>,
9287 carry_seats: bool,
9291 pauses: Option<(&'a Pause, &'a Pause)>,
9294}
9295
9296fn repository_wait(
9298 state: &RunState,
9299 pauses: Option<(&Pause, &Pause)>,
9300) -> crate::repo_lock::WaitPolicy {
9301 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9302 let inherited = crate::repo_lock::WaitPolicy::current();
9303 crate::repo_lock::WaitPolicy::new(
9304 Duration::from_secs(state.config.graph.timeout_resource_wait),
9305 move || {
9306 inherited.cancelled()
9307 || pauses
9308 .as_ref()
9309 .is_some_and(|(a, b)| a.parked() || b.parked())
9310 },
9311 )
9312}
9313
9314fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9315 if error.is::<crate::repo_lock::ResourceBlocked>() {
9316 state.resource_block = Some(crate::run::ResourceBlock {
9317 node: "repository".into(),
9318 kind: if error.to_string().contains("disk pressure") {
9319 "disk pressure"
9320 } else {
9321 "repository coordination"
9322 }
9323 .into(),
9324 detail: error.to_string(),
9325 at: Timestamp::now(),
9326 });
9327 state.event("resource", error.to_string());
9328 }
9329}
9330
9331async fn coordinate_operation(
9332 state: &mut RunState,
9333 cwd: &Path,
9334 wait: &crate::repo_lock::WaitPolicy,
9335) -> Result<Option<crate::repo_lock::SeatLease>> {
9336 let result =
9337 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9338 .await;
9339 if let Err(e) = &result {
9340 record_repository_block(state, e);
9341 }
9342 result
9343}
9344
9345async fn run_one(
9347 job: SeatJob,
9348 sem: Arc<Semaphore>,
9349 ctx: &WaveCtx<'_>,
9350 state: &mut RunState,
9351 attempt: usize,
9352) -> (SeatState, AgentOutcome) {
9353 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9354 .await
9355 .pop()
9356 .expect("one job in, one result out");
9357 (seat, out)
9358}
9359
9360async fn wave(
9366 jobs: Vec<SeatJob>,
9367 sem: Arc<Semaphore>,
9368 ctx: &WaveCtx<'_>,
9369 state: &mut RunState,
9370 attempt: usize,
9371) -> Vec<(usize, SeatState, AgentOutcome)> {
9372 let WaveCtx {
9373 run,
9374 node,
9375 prompts,
9376 cache,
9377 round,
9378 carry_seats: _,
9379 pauses,
9380 } = *ctx;
9381 for job in &jobs {
9382 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9383 }
9384 if let Err(e) = state.save() {
9385 tracing::warn!("could not persist in-progress seats: {e:#}");
9390 }
9391 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9407 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9411 let cache_guard = if let Some(cache_dir) = cache {
9412 if jobs_had_a_writer {
9413 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9414 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9415 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9416 {
9417 Ok(Some(g)) => Some(g),
9418 Ok(None) => {
9421 state.resource_block = Some(crate::run::ResourceBlock {
9422 node: node.to_owned(),
9423 kind: "shared build cache".to_owned(),
9424 detail: "the run is parking".to_owned(),
9425 at: Timestamp::now(),
9426 });
9427 return block_wave(
9428 jobs,
9429 state,
9430 node,
9431 attempt,
9432 "parked while waiting for the shared build cache",
9433 );
9434 }
9435 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9438 Err(e) => {
9439 let why = format!("{e:#}");
9443 state.resource_block = Some(crate::run::ResourceBlock {
9444 node: node.to_owned(),
9445 kind: "shared build cache".to_owned(),
9446 detail: why.clone(),
9447 at: Timestamp::now(),
9448 });
9449 state.event(
9450 "resource",
9451 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9452 );
9453 return block_wave(jobs, state, node, attempt, &why);
9454 }
9455 }
9456 } else {
9457 None
9458 }
9459 } else {
9460 None
9461 };
9462 state.seats_spawning(node, attempt);
9465 if let Err(e) = state.save() {
9466 tracing::warn!("could not persist the start of a wave: {e:#}");
9467 }
9468 let slot_limit = resource_limit
9472 + jobs
9473 .iter()
9474 .map(|j| j.timeout)
9475 .max()
9476 .unwrap_or_default()
9477 .saturating_mul(jobs.len() as u32);
9478 let mut set = tokio::task::JoinSet::new();
9479 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9482 let overlay = prompts.overlay(node);
9483 let repository_wait = repository_wait(state, pauses);
9484 for (i, mut job) in jobs.into_iter().enumerate() {
9485 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9486 if cache.is_some() {
9487 job.prompt.push('\n');
9488 job.prompt
9489 .push_str(&prompt::build_cache_note(node, job.allow_write));
9490 }
9491 let sem = Arc::clone(&sem);
9492 let run = run.to_owned();
9493 let node = node.to_owned();
9494 let attachments = if node == "implement" {
9497 state.attachments.clone()
9498 } else {
9499 Vec::new()
9500 };
9501 let cache = cache
9512 .filter(|_| job.allow_write && cache_guard.is_some())
9513 .map(Path::to_path_buf);
9514 let spawned_tx = spawned_tx.clone();
9515 let repository_wait = repository_wait.clone();
9516 let coordination = state.config.coordination.clone();
9517 let fetched_base = state.fetched_base.clone();
9518 set.spawn(async move {
9519 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9523 Ok(p) => p,
9524 Err(_) => {
9525 return (
9526 i,
9527 job.seat,
9528 AgentOutcome::ResourceBlocked(format!(
9529 "no parallel slot became free within {}s",
9530 slot_limit.as_secs()
9531 )),
9532 false,
9533 );
9534 }
9535 };
9536 spawned_tx.send(job.seat.key.clone()).ok();
9537 let mut seat = job.seat;
9538 let out = repository_wait
9539 .scope(agent::invoke(
9540 &job.spec,
9541 &mut seat,
9542 &Invocation {
9543 scope: crate::agent::InvocationScope::Repository,
9544 coordination: coordination.as_ref(),
9545 fetched_base: fetched_base.as_ref(),
9546 cwd: &job.cwd,
9547 prompt: &job.prompt,
9548 timeout: job.timeout,
9549 allow_write: job.allow_write,
9550 unsandboxed: false,
9551 sessions: job.sessions,
9552 artifacts: &job.artifacts,
9553 stem: &job.stem,
9554 run: &run,
9555 node: &node,
9556 cache_dir: cache.as_deref(),
9557 attachments: &attachments,
9558 writable: &[],
9559 },
9560 ))
9561 .await;
9562 let interrupted = out
9563 .as_ref()
9564 .err()
9565 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9566 let out = match out {
9567 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9568 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9569 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9577 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9578 Ok(o) => AgentOutcome::Failed(format!(
9579 "process exited with no usable output ({})",
9580 crate::notices::exit_status(o.exit_code)
9581 )),
9582 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9583 AgentOutcome::ResourceBlocked(e.to_string())
9584 }
9585 Err(e) => AgentOutcome::Failed(e.to_string()),
9586 };
9587 (i, seat, out, interrupted)
9588 });
9589 }
9590 drop(spawned_tx);
9591 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9592 loop {
9593 let joined = tokio::select! {
9594 Some(key) = spawned_rx.recv() => {
9595 state.seat_spawned(&key);
9597 if let Err(e) = state.save() {
9598 tracing::warn!("could not persist a seat's start: {e:#}");
9599 }
9600 continue;
9601 }
9602 joined = set.join_next() => joined,
9603 };
9604 let Some(joined) = joined else {
9605 break;
9606 };
9607 let (i, seat, out, interrupted) = match joined {
9608 Ok(v) => v,
9609 Err(e) => {
9613 tracing::error!("agent task panicked: {e}");
9614 continue;
9615 }
9616 };
9617 state.seat_finished(&seat.key);
9618 if interrupted
9619 && node == "implement"
9620 && let Some(label) = state
9621 .candidates
9622 .iter()
9623 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9624 .map(|candidate| candidate.label)
9625 {
9626 state.candidate_preservation.insert(
9627 label,
9628 "interrupted implementation awaiting persistence".into(),
9629 );
9630 state.interrupted_implementations.insert(label);
9631 }
9632 if let AgentOutcome::ResourceBlocked(why) = &out
9633 && state.resource_block.is_none()
9634 {
9635 state.resource_block = Some(crate::run::ResourceBlock {
9636 node: node.to_owned(),
9637 kind: if why.contains("disk pressure") {
9638 "disk pressure"
9639 } else if why.contains("repository coordination") {
9640 "repository coordination"
9641 } else {
9642 "parallel slot"
9643 }
9644 .to_owned(),
9645 detail: why.clone(),
9646 at: Timestamp::now(),
9647 });
9648 let activity = if interrupted {
9649 "agent execution interrupted"
9650 } else {
9651 "no agent was started"
9652 };
9653 state.event("resource", format!("{node}: {activity} - {why}"));
9654 }
9655 record_jobs(state, node, round, &seat.key, &out);
9656 if let Err(e) = state.save() {
9657 tracing::warn!("could not persist a seat's completion: {e:#}");
9658 }
9659 if collected.len() <= i {
9660 collected.resize_with(i + 1, || None);
9661 }
9662 collected[i] = Some((i, seat, out));
9663 }
9664 if state
9670 .active
9671 .values()
9672 .any(|a| a.node == node && a.attempt == attempt)
9673 {
9674 state
9675 .active
9676 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9677 if let Err(e) = state.save() {
9678 tracing::warn!("could not persist the end of a wave: {e:#}");
9679 }
9680 }
9681 if let Some(cache_dir) = cache
9688 && jobs_had_a_writer
9689 {
9690 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9691 }
9692 if let Some(guard) = cache_guard {
9693 guard.release();
9694 }
9695 collected.into_iter().flatten().collect()
9696}
9697
9698fn record_jobs(
9709 state: &mut RunState,
9710 node: &str,
9711 round: Option<usize>,
9712 seat: &str,
9713 out: &AgentOutcome,
9714) {
9715 let commands: &[agent::CommandEvidence] = match out {
9716 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9717 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9718 };
9719 let checked_at = Timestamp::now();
9720 for c in commands {
9721 state.jobs.push(JobRecord {
9722 node: node.to_owned(),
9723 round,
9724 seat: seat.to_owned(),
9725 id: c.id.clone(),
9726 description: c.description.clone(),
9727 checked_at,
9728 status: match c.exit_code {
9729 Some(0) => JobStatus::Completed,
9730 Some(_) => JobStatus::Failed,
9731 None => JobStatus::Unknown,
9732 },
9733 exit_code: c.exit_code,
9734 result_summary: c.result_summary.clone(),
9735 source: c.source.clone(),
9736 });
9737 }
9738}
9739
9740fn round_is_clean(
9761 blocking: usize,
9762 e2e_ok: bool,
9763 answered: usize,
9764 expected: usize,
9765 quota_missing: usize,
9766 policy: IncompleteReviewPolicy,
9767) -> bool {
9768 if blocking != 0 || !e2e_ok {
9769 return false;
9770 }
9771 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9772 return true;
9773 }
9774 answered > 0 && expected - answered <= quota_missing
9775}
9776
9777pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9801 if max_rounds == 0
9802 || reviews.last().is_some_and(|r| {
9803 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9804 })
9805 {
9806 return Some(RunStatus::Gating);
9807 }
9808 let last = reviews.last()?;
9809 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9810 if reviews.len() < max_rounds && !stagnant {
9811 return None;
9812 }
9813 if last.incomplete() && last.blocking == 0 {
9814 return Some(RunStatus::Blocked);
9815 }
9816 if last.e2e_status() == E2eStatus::ResourceBlocked
9817 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9818 {
9819 return None;
9820 }
9821 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9822 RunStatus::Gating
9823 } else {
9824 RunStatus::Blocked
9825 })
9826}
9827
9828pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9843 if nudged {
9844 (full / 4).max(Duration::from_secs(120)).min(full)
9845 } else {
9846 full
9847 }
9848}
9849
9850#[allow(clippy::too_many_arguments)]
9879async fn ask_json_wave<T>(
9880 jobs: Vec<SeatJob>,
9881 sem: Arc<Semaphore>,
9882 retries: usize,
9883 roster: &[AgentSpec],
9884 ctx: &WaveCtx<'_>,
9885 losses: &mut Vec<QuotaLoss>,
9886 state: &mut RunState,
9887 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9888) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9889where
9890 T: serde::de::DeserializeOwned + Send + 'static,
9891{
9892 ask_wave_with(
9893 jobs,
9894 sem,
9895 retries,
9896 roster,
9897 ctx,
9898 losses,
9899 state,
9900 &|text: &str| {
9901 let v = verdict::extract_json::<T>(text)?;
9902 validate(&v)?;
9903 Ok(v)
9904 },
9905 )
9906 .await
9907}
9908
9909#[allow(clippy::too_many_arguments)]
9918async fn ask_wave_with<T>(
9919 jobs: Vec<SeatJob>,
9920 sem: Arc<Semaphore>,
9921 retries: usize,
9922 roster: &[AgentSpec],
9923 ctx: &WaveCtx<'_>,
9924 losses: &mut Vec<QuotaLoss>,
9925 state: &mut RunState,
9926 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9927) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9928where
9929 T: Send + 'static,
9930{
9931 let n = jobs.len();
9932 let originals: Vec<SeatJob> = jobs;
9933 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9934 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9935 let mut nudges: Vec<usize> = vec![0; n];
9942 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9946 let mut tried: Vec<BTreeSet<String>> = specs
9947 .iter()
9948 .map(|s| BTreeSet::from([s.id.clone()]))
9949 .collect();
9950 let starts: Vec<usize> = specs
9951 .iter()
9952 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9953 .collect();
9954 let carry = ctx.carry_seats && !roster.is_empty();
9955 let carried: Vec<BTreeSet<String>> = originals
9958 .iter()
9959 .map(|j| {
9960 state
9961 .seat_history
9962 .get(&j.seat.key)
9963 .filter(|_| carry)
9964 .map(|h| h.failed.clone())
9965 .unwrap_or_default()
9966 })
9967 .collect();
9968 let mut prev: Vec<Option<FailClass>> = originals
9969 .iter()
9970 .map(|j| {
9971 state
9972 .seat_history
9973 .get(&j.seat.key)
9974 .filter(|_| carry)
9975 .and_then(|h| h.last_fail.clone())
9976 })
9977 .collect();
9978 let next_agent =
9979 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9980 let others: BTreeSet<String> = specs
9981 .iter()
9982 .enumerate()
9983 .filter(|(j, _)| *j != i)
9984 .map(|(_, s)| s.id.clone())
9985 .collect();
9986 pick_successor(
9987 roster,
9988 starts[i],
9989 tried,
9990 carry.then(|| &carried[i]),
9991 &others,
9992 )
9993 .cloned()
9994 };
9995 let mut fresh: Vec<Option<String>> = vec![None; n];
9996 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9997 let mut pending: Vec<usize> = (0..n).collect();
9998
9999 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10004 for round in 0..max_rounds {
10005 if pending.is_empty() {
10006 break;
10007 }
10008 let mut batch = Vec::with_capacity(pending.len());
10009 let mut renudged: Vec<&str> = Vec::new();
10010 for &i in &pending {
10011 let src = &originals[i];
10012 let full: &str = match &src.handover {
10017 Some(h) if specs[i].id != src.spec.id => h,
10018 _ => &src.prompt,
10019 };
10020 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10023 (
10024 full.to_owned(),
10025 src.timeout,
10026 format!("{}-{word}-{}", src.stem, specs[i].id),
10027 )
10028 } else if nudges[i] == 0 {
10029 (full.to_owned(), src.timeout, src.stem.clone())
10030 } else {
10031 renudged.push(src.seat.key.as_str());
10032 let why = done[i]
10033 .as_ref()
10034 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10035 .unwrap_or_else(|| "no parsable answer".to_owned());
10036 let nudge = prompt::nudge(&why);
10037 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10038 let prompt = if nudged {
10039 nudge
10040 } else {
10041 format!("{full}\n\n---\n\n{nudge}")
10042 };
10043 (
10044 prompt,
10045 retry_budget(src.timeout, nudged),
10046 format!("{}-retry{}", src.stem, nudges[i]),
10047 )
10048 };
10049 batch.push(SeatJob {
10050 spec: specs[i].clone(),
10051 seat: seats[i].clone(),
10052 cwd: src.cwd.clone(),
10053 prompt,
10054 timeout,
10055 allow_write: src.allow_write,
10056 sessions: src.sessions,
10057 artifacts: src.artifacts.clone(),
10058 stem,
10059 handover: None,
10060 });
10061 }
10062
10063 if !renudged.is_empty() {
10064 state.event(
10065 ctx.node,
10066 format!("retry {round}: re-asking {}", renudged.join(", ")),
10067 );
10068 }
10069 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10070 let mut still = Vec::new();
10071 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10072 seats[i] = seat;
10073 if let AgentOutcome::ResourceBlocked(why) = &out {
10076 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10077 continue;
10078 }
10079 let class = FailClass::of(&out);
10080 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10083 && nudges[i] < retries
10084 && !roster.is_empty();
10085 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10086 let next = should_hand_over(prev[i].as_ref(), &cur)
10087 .then(|| next_agent(i, &tried[i], &specs))
10088 .flatten();
10089 if carry {
10090 let h = state
10091 .seat_history
10092 .entry(originals[i].seat.key.clone())
10093 .or_default();
10094 h.failed.insert(specs[i].id.clone());
10095 h.last_fail = Some(cur.clone());
10096 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10097 h.last_ok = None;
10098 }
10099 if let Err(e) = state.save() {
10102 tracing::warn!("could not persist a seat's failure history: {e:#}");
10103 }
10104 }
10105 if let Some(next) = next {
10106 record_handover(
10107 state,
10108 ctx.node,
10109 &originals[i].seat.key,
10110 &specs[i].id,
10111 &next.id,
10112 &cur,
10113 &fail_reason(&out),
10114 );
10115 tried[i].insert(next.id.clone());
10116 prev[i] = Some(cur.clone());
10117 seats[i] =
10118 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10119 specs[i] = next;
10120 fresh[i] = Some(cur.stem_word().to_owned());
10121 nudges[i] = 0;
10122 done[i] = Some(Err(anyhow::anyhow!(
10123 "handed over after: {}",
10124 fail_reason(&out)
10125 )));
10126 still.push(i);
10127 continue;
10128 }
10129 }
10130 if carry
10131 && !nudge_first
10132 && let Some(cur) = class.clone()
10133 {
10134 let h = state
10136 .seat_history
10137 .entry(originals[i].seat.key.clone())
10138 .or_default();
10139 h.failed.insert(specs[i].id.clone());
10140 h.last_fail = Some(cur);
10141 }
10142 let parsed = match out {
10143 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10144 AgentOutcome::Quota(o) => {
10145 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10146 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10147 }
10148 AgentOutcome::Dropped(o) => {
10152 let why = o
10153 .dropped
10154 .as_ref()
10155 .map(|d| d.why.as_str())
10156 .unwrap_or("the CLI ended the stream without delivering its answer");
10157 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10158 }
10159 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10160 Err(anyhow::anyhow!(e))
10161 }
10162 };
10163 let quota = class == Some(FailClass::Quota);
10164 let failed = parsed.is_err();
10165 done[i] = Some(parsed);
10166 if carry && !failed {
10167 let h = state
10168 .seat_history
10169 .entry(originals[i].seat.key.clone())
10170 .or_default();
10171 h.failed.remove(&specs[i].id);
10172 h.last_ok = Some(specs[i].id.clone());
10173 h.last_fail = None;
10174 }
10175 let agent_failure = class.is_some()
10183 && !nudge_first
10184 && !roster.is_empty()
10185 && next_agent(i, &tried[i], &specs).is_some();
10186 if failed && !quota && !agent_failure && nudges[i] < retries {
10187 nudges[i] += 1;
10188 still.push(i);
10189 }
10190 }
10191 pending = still;
10192 }
10193
10194 for (i, q) in last_quota.into_iter().enumerate() {
10197 if let Some(reset) = q {
10198 losses.push(QuotaLoss {
10199 seat: originals[i].seat.key.clone(),
10200 node: ctx.node.to_owned(),
10201 at: Timestamp::now(),
10202 reset,
10203 });
10204 }
10205 }
10206
10207 seats
10208 .into_iter()
10209 .zip(done)
10210 .zip(nudges)
10211 .map(|((seat, res), attempts)| {
10212 (
10213 seat,
10214 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10215 attempts,
10216 )
10217 })
10218 .collect()
10219}
10220
10221fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10223 results
10224 .iter()
10225 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10226}
10227
10228fn block_wave(
10232 jobs: Vec<SeatJob>,
10233 state: &mut RunState,
10234 node: &str,
10235 attempt: usize,
10236 why: &str,
10237) -> Vec<(usize, SeatState, AgentOutcome)> {
10238 state.resource_wait = None;
10239 state
10240 .active
10241 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10242 if let Err(e) = state.save() {
10243 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10244 }
10245 jobs.into_iter()
10246 .enumerate()
10247 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10248 .collect()
10249}
10250
10251async fn acquire_cache_lease(
10264 state: &mut RunState,
10265 cache_dir: &Path,
10266 owner: &crate::cache::Owner,
10267 budget: Duration,
10268 context: &str,
10269 cancel: &(dyn Fn() -> bool + Sync),
10270) -> Result<Option<crate::cache::Guard>> {
10271 let home = crate::run::home();
10272 let started = Instant::now();
10273 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10274 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10275 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10276 Err(e) => {
10277 state.event(
10278 "verify",
10279 format!("{context}: could not check the shared build cache: {e:#}"),
10280 );
10281 if let Err(e2) = state.save() {
10282 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10283 }
10284 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10285 }
10286 };
10287 state.event(
10288 "verify",
10289 format!(
10290 "{context}: waiting for the shared build cache at {} ({})",
10291 cache_dir.display(),
10292 busy.describe()
10293 ),
10294 );
10295 state.resource_wait = Some(crate::run::ResourceWait {
10298 node: context.to_owned(),
10299 kind: "shared build cache".to_owned(),
10300 holder: busy.describe(),
10301 since: Timestamp::now(),
10302 limit_secs: budget.as_secs(),
10303 });
10304 if let Err(e) = state.save() {
10305 tracing::warn!("could not persist a cache wait: {e:#}");
10306 }
10307 let remaining = budget.saturating_sub(started.elapsed());
10308 let waited = crate::cache::wait_for_unless(
10309 &home,
10310 cache_dir,
10311 owner,
10312 remaining,
10313 Duration::from_secs(5),
10314 cancel,
10315 )
10316 .await;
10317 state.resource_wait = None;
10318 match waited {
10319 Ok(g) => Ok(g),
10320 Err(e) => {
10321 state.event("verify", format!("{context}: {e:#}"));
10322 if let Err(e2) = state.save() {
10323 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10324 }
10325 Err(e)
10326 }
10327 }
10328}
10329
10330#[derive(Debug)]
10335struct CacheCheckError(String);
10336
10337impl std::fmt::Display for CacheCheckError {
10338 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10339 f.write_str(&self.0)
10340 }
10341}
10342
10343impl std::error::Error for CacheCheckError {}
10344
10345#[allow(clippy::too_many_arguments)]
10366async fn with_cache_lease<'s, F, Fut>(
10367 state: &'s mut RunState,
10368 cache_dir: Option<&Path>,
10369 node: &str,
10370 seat: &str,
10371 worktree: &Path,
10372 head: &str,
10373 budget: Duration,
10374 context: &str,
10375 wait: &crate::repo_lock::WaitPolicy,
10376 body: F,
10377) -> Result<(Vec<CommandOutcome>, bool)>
10378where
10379 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10380 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10381{
10382 let started = Instant::now();
10385 let home = crate::run::home();
10386 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10387 let guard = if let Some(cache_dir) = cache_dir {
10388 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10389 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10390 wait.cancelled()
10391 })
10392 .await
10393 .and_then(|g| {
10394 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10395 }) {
10396 Ok(g) => Some(g),
10397 Err(e) => {
10398 return Ok((
10399 vec![CommandOutcome {
10400 command: "(waiting for the shared build cache)".into(),
10401 code: None,
10402 output_tail: e.to_string(),
10403 duration_ms: started.elapsed().as_millis() as u64,
10404 resource_blocked: true,
10405 }],
10406 false,
10407 ));
10408 }
10409 }
10410 } else {
10411 None
10412 };
10413 let _coordination = coordinate_operation(state, worktree, wait).await?;
10414 if let Some(cache_dir) = cache_dir {
10415 let identity = crate::cache::Identity::new(worktree, head);
10416 if let Err(e) = crate::cache::ensure_fresh(
10417 &home,
10418 cache_dir,
10419 &identity,
10420 &state.config.verify.cargo_manifests(),
10421 ) {
10422 state.event(
10423 "verify",
10424 format!(
10425 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10426 worktree.display(),
10427 short(head)
10428 ),
10429 );
10430 if let Some(guard) = guard {
10431 guard.release();
10432 }
10433 return Ok((
10434 vec![CommandOutcome {
10435 command: "(confirming the shared build cache is fresh)".into(),
10436 code: None,
10437 output_tail: e.to_string(),
10438 duration_ms: started.elapsed().as_millis() as u64,
10439 resource_blocked: true,
10440 }],
10441 false,
10442 ));
10443 }
10444 }
10445 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10446 if !timed_out_pids.is_empty() {
10447 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10448 }
10449 drop(_coordination);
10452 if let Some(guard) = guard {
10453 guard.release();
10454 }
10455 Ok((outcomes, retried))
10456}
10457
10458async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10470 wait_for_pids_with(
10471 pids,
10472 crate::proc::pid_alive,
10473 LEASE_RELEASE_POLL,
10474 LEASE_RELEASE_MAX_WAIT,
10475 )
10476 .await;
10477}
10478
10479async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10485 pids: &[u32],
10486 alive: F,
10487 poll: Duration,
10488 max_wait: Duration,
10489) {
10490 let deadline = Instant::now() + max_wait;
10491 loop {
10492 if pids.iter().all(|&pid| !alive(pid)) {
10493 return;
10494 }
10495 if Instant::now() >= deadline {
10496 return;
10497 }
10498 tokio::time::sleep(poll).await;
10499 }
10500}
10501
10502fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10509 outcomes.iter().any(|o| o.resource_blocked)
10510}
10511
10512pub(crate) enum GateFix {
10514 Retry,
10516 Stop,
10519 Defer,
10522}
10523
10524fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10532 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10533 red.peek().is_some()
10534 && red.all(|o| {
10535 !o.resource_blocked
10536 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10537 && !o.output_tail.trim().is_empty()
10538 })
10539}
10540
10541fn e2e_outcome_label(o: &CommandOutcome) -> String {
10545 if o.ok() {
10546 return "pass".to_owned();
10547 }
10548 let reason = if o.build_failed() {
10549 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10550 } else {
10551 format!("FAIL ({:?})", o.code)
10552 };
10553 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10554}
10555
10556async fn run_e2e_with_retry(
10564 state: &mut RunState,
10565 shell: &[String],
10566 commands: &[String],
10567 worktree: &Path,
10568 timeout: Duration,
10569 context: &str,
10570) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10571 let (mut e2e, mut timed_out_pids) = run_commands(
10572 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10573 )
10574 .await;
10575 for o in &e2e {
10576 state.event(
10577 "verify",
10578 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10579 );
10580 }
10581 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10585 if verify_retried {
10586 state.event(
10587 "verify",
10588 format!(
10589 "{context}: verify could not build/link, not a test result — retrying once \
10590 before concluding"
10591 ),
10592 );
10593 let retried = run_commands(
10594 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10595 )
10596 .await;
10597 e2e = retried.0;
10598 timed_out_pids.extend(retried.1);
10601 for o in &e2e {
10602 state.event(
10603 "verify",
10604 format!(
10605 "{context}: retry `{}` -> {}",
10606 o.command,
10607 e2e_outcome_label(o)
10608 ),
10609 );
10610 }
10611 }
10612 (e2e, verify_retried, timed_out_pids)
10613}
10614
10615#[allow(clippy::too_many_arguments)]
10631async fn run_commands(
10632 state: &mut RunState,
10633 node: &str,
10634 task: &str,
10635 attempt: usize,
10636 shell: &[String],
10637 commands: &[String],
10638 cwd: &Path,
10639 timeout: Duration,
10640) -> (Vec<CommandOutcome>, Vec<u32>) {
10641 if commands.is_empty() {
10642 return (Vec::new(), Vec::new());
10647 }
10648 let mut out = Vec::new();
10649 let mut timed_out_pids = Vec::new();
10650 let total = commands.len();
10651 for (idx, command) in commands.iter().enumerate() {
10652 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10653 if let Err(e) = state.save() {
10654 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10655 }
10656 let started = Instant::now();
10657 let mut cmd = tokio::process::Command::new(&shell[0]);
10658 cmd.quiet();
10659 cmd.args(&shell[1..])
10660 .arg(command)
10661 .current_dir(cwd)
10662 .stdin(std::process::Stdio::null())
10663 .stdout(std::process::Stdio::piped())
10664 .stderr(std::process::Stdio::piped())
10665 .kill_on_drop(true);
10666 let mut outputs = Vec::new();
10667 let protected = (|| -> Result<()> {
10668 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10669 outputs.push(guard);
10670 }
10671 if let Some(cache) = state.config.cache_dir() {
10672 outputs.push(crate::retention::OutputLease::cache(
10673 &crate::run::home(),
10674 &cache,
10675 false,
10676 )?);
10677 }
10678 Ok(())
10679 })();
10680 if let Err(e) = protected {
10681 out.push(CommandOutcome {
10682 command: command.clone(),
10683 code: None,
10684 output_tail: format!("output ownership unavailable: {e:#}"),
10685 duration_ms: 0,
10686 resource_blocked: true,
10687 });
10688 continue;
10689 }
10690 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10691 let (code, body) = match spawned {
10692 Ok(mut running) => {
10693 let pid = running.child().id();
10694 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10695 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10696 let mut paths = vec![cwd.to_owned()];
10697 if let Some(cache) = state.config.cache_dir() {
10698 paths.push(cache);
10699 }
10700 let mut pressure = None;
10701 let status = tokio::select! {
10702 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10703 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10704 pressure = Some(reason); None
10705 }
10706 };
10707 let (code, failure) = match status {
10708 Some(Ok(Ok(status))) => (
10709 status.code(),
10710 status
10711 .code()
10712 .is_none()
10713 .then(|| format!("command terminated: {status}")),
10714 ),
10715 Some(Ok(Err(error))) => {
10716 (None, Some(format!("failed waiting for command: {error}")))
10717 }
10718 Some(Err(_)) => {
10719 if !running.coordinated()
10723 && let Some(pid) = pid
10724 {
10725 timed_out_pids.push(pid);
10726 }
10727 (
10728 None,
10729 Some(format!("timed out after {}s", timeout.as_secs())),
10730 )
10731 }
10732 None => (None, None),
10733 };
10734 running.finish().await;
10735 if let Some(reason) = pressure {
10736 state.resource_block = Some(crate::run::ResourceBlock {
10737 node: node.to_owned(),
10738 kind: "disk pressure".to_owned(),
10739 detail: reason.clone(),
10740 at: Timestamp::now(),
10741 });
10742 out.push(CommandOutcome {
10743 command: command.clone(),
10744 code: None,
10745 output_tail: reason,
10746 duration_ms: started.elapsed().as_millis() as u64,
10747 resource_blocked: true,
10748 });
10749 break;
10750 }
10751 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10752 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10753 if let Some(failure) = failure {
10754 body.push('\n');
10755 body.push_str(&failure);
10756 }
10757 (code, body)
10758 }
10759 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10760 };
10761 out.push(CommandOutcome {
10762 command: command.clone(),
10763 code,
10764 output_tail: tail(&body, OUTPUT_TAIL),
10765 duration_ms: started.elapsed().as_millis() as u64,
10766 resource_blocked: false,
10767 });
10768 }
10769 state.task_finished(task);
10770 if let Err(e) = state.save() {
10771 tracing::warn!("could not persist the end of {task}: {e:#}");
10772 }
10773 (out, timed_out_pids)
10774}
10775
10776fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10788 let repo = repo.display();
10789 match style {
10790 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10791 MergeStyle::Squash => {
10792 let subject = message
10795 .lines()
10796 .next()
10797 .unwrap_or(branch)
10798 .replace(['\\', '"', '$', '`'], "");
10799 format!(
10800 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10801 )
10802 }
10803 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10804 }
10805}
10806
10807const PR_TITLE_MAX: usize = 240;
10818
10819fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10830 let line = queue::first_line(text)?;
10831 let chars: Vec<char> = line.chars().collect();
10832 let (kept, mark) = if chars.len() <= max {
10833 (line.to_owned(), "")
10834 } else if let Some(end) = sentence_end(&chars, max) {
10835 (chars[..end].iter().collect::<String>(), "")
10836 } else {
10837 let room = max.saturating_sub(3);
10838 let cut = if chars[room].is_whitespace() {
10841 room
10842 } else {
10843 chars[..room]
10844 .iter()
10845 .rposition(|c| c.is_whitespace())
10846 .unwrap_or(room)
10847 };
10848 let head: String = chars[..cut].iter().collect();
10849 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10850 (head.to_owned(), "...")
10851 };
10852 if kept.is_empty() {
10853 return None;
10854 }
10855 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10856 return None;
10857 }
10858 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10859 return None;
10860 }
10861 Some(format!("{kept}{mark}"))
10862}
10863
10864fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10868 const MIN: usize = 20;
10869 for i in MIN..max.min(chars.len()) {
10870 if !matches!(chars[i], '.' | '!' | '?') {
10871 continue;
10872 }
10873 let Some(&next) = chars.get(i + 1) else {
10874 continue;
10875 };
10876 if !next.is_whitespace() {
10877 continue;
10878 }
10879 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10880 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10881 continue;
10882 }
10883 let word: String = chars[..i]
10884 .iter()
10885 .rev()
10886 .take_while(|c| !c.is_whitespace())
10887 .collect::<Vec<_>>()
10888 .into_iter()
10889 .rev()
10890 .collect();
10891 let word = word.to_ascii_lowercase();
10892 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10893 continue;
10894 }
10895 let end = chars[..i]
10896 .iter()
10897 .rposition(|c| !c.is_whitespace())
10898 .map_or(i, |p| p + 1);
10899 return Some(end);
10900 }
10901 None
10902}
10903
10904struct PrMessage {
10909 title: String,
10910 body: String,
10911}
10912
10913impl PrMessage {
10914 fn commit_message(&self) -> String {
10918 format!("{}\n\n{}", self.title, self.body)
10919 }
10920}
10921
10922fn title_marker(line: &str) -> Option<&str> {
10924 let line = line.trim();
10925 let head = line.get(..6)?;
10926 head.eq_ignore_ascii_case("title:")
10927 .then(|| line[6..].trim())
10928}
10929
10930fn summary_title(summary: &str) -> Option<String> {
10935 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10936 let raw = title_marker(first)?;
10937 if raw.is_empty() {
10938 return None;
10939 }
10940 let title = queue::title_from(raw, PR_TITLE_MAX);
10941 let lower = title.to_ascii_lowercase();
10942 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10943 return None;
10944 }
10945 Some(title)
10946}
10947
10948const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10950
10951pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10953
10954pub(crate) fn is_review_run(state: &RunState) -> bool {
10958 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10959}
10960
10961fn review_title(state: &RunState) -> Option<String> {
10966 english_subject(
10967 state.reviewed_commits.as_ref()?.first()?,
10968 state.config.graph.github_english(),
10969 )
10970}
10971
10972fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10975 let raw = raw.trim();
10976 if raw.is_empty() {
10977 return None;
10978 }
10979 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10980 return None;
10981 }
10982 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10983 return None;
10984 }
10985 let title = queue::title_from(raw, PR_TITLE_MAX);
10986 let lower = title.to_ascii_lowercase();
10987 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10988 return None;
10989 }
10990 Some(title)
10991}
10992
10993#[derive(Debug, Clone, PartialEq, Eq)]
10996struct BranchFacts {
10997 commits: Vec<(String, String)>,
10999 stat: String,
11001}
11002
11003const STAT_FILE_LINES: usize = 25;
11005const COMMITS_MAX_CHARS: usize = 20_000;
11007
11008async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11011 let commits = git::commit_log(repo, base, branch).await.ok()?;
11012 if commits.is_empty() {
11013 return None;
11014 }
11015 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11016 let lines: Vec<&str> = stat.lines().collect();
11017 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11018 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11019 let more = format!(" ... {omitted} more file(s)");
11020 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11021 kept.push(&more);
11022 kept.push(lines[lines.len() - 1]);
11023 kept.join("\n")
11024 } else {
11025 lines.join("\n")
11026 };
11027 Some(BranchFacts { commits, stat })
11028}
11029
11030fn markdown_safe(text: &str) -> String {
11033 text.replace("</details>", "</details>")
11034 .replace("\x60\x60\x60", "~~~")
11035}
11036
11037fn neutral_title(state: &RunState, winner: char) -> String {
11038 format!(
11039 "chore: land candidate {} of run {}",
11040 winner.to_ascii_uppercase(),
11041 state.id
11042 )
11043}
11044
11045pub(crate) async fn repair_landing_gate(
11048 state: &mut RunState,
11049 winner: &Candidate,
11050 outcomes: &[CommandOutcome],
11051) -> Result<GateFix> {
11052 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11053 return Ok(GateFix::Stop);
11054 }
11055 let mut runner = Runner {
11056 roles: state.config.resolve_roles()?,
11057 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11058 state: state.clone(),
11059 pause: Pause::new(),
11060 interrupt: Pause::new(),
11061 };
11062 let result = runner.gate_fix_round(winner, outcomes).await;
11063 *state = runner.state;
11064 result
11065}
11066
11067pub(crate) async fn check_gate_repair_e2e(
11070 state: &mut RunState,
11071 winner: &Candidate,
11072 wait: &crate::repo_lock::WaitPolicy,
11073) -> Result<GateFix> {
11074 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11075 return Ok(GateFix::Retry);
11076 }
11077 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11078 if state
11079 .gate_repair_e2e
11080 .as_ref()
11081 .is_some_and(|r| r.head == head && r.passed)
11082 {
11083 return Ok(GateFix::Retry);
11084 }
11085 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11086 head: head.clone(),
11087 passed: false,
11088 outcomes: Vec::new(),
11089 });
11090 state.save()?;
11091 let commands = state.config.verify.e2e.clone();
11092 let outcomes = if commands.is_empty() {
11093 Vec::new()
11094 } else {
11095 let shell = state.config.shell();
11096 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11097 let cache_dir = state.config.cache_dir();
11098 let (outcomes, _) = with_cache_lease(
11099 state,
11100 cache_dir.as_deref(),
11101 "e2e",
11102 "e2e",
11103 &winner.worktree,
11104 &head,
11105 timeout,
11106 "gate repair e2e",
11107 wait,
11108 |state, budget| async move {
11109 run_e2e_with_retry(
11110 state,
11111 &shell,
11112 &commands,
11113 &winner.worktree,
11114 budget,
11115 "gate repair e2e",
11116 )
11117 .await
11118 },
11119 )
11120 .await?;
11121 outcomes
11122 };
11123 let deferred = verify_inconclusive(&outcomes);
11124 let passed = outcomes.iter().all(CommandOutcome::ok)
11125 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11126 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11127 head,
11128 passed,
11129 outcomes,
11130 });
11131 state.save()?;
11132 if deferred {
11133 Ok(GateFix::Defer)
11134 } else if !passed {
11135 state.event(
11136 "gate",
11137 "verify.e2e failed after gate repair; nothing published",
11138 );
11139 Ok(GateFix::Stop)
11140 } else {
11141 Ok(GateFix::Retry)
11142 }
11143}
11144
11145pub(crate) async fn run_landing_gate(
11147 state: &mut RunState,
11148 winner: &Candidate,
11149 wait: &crate::repo_lock::WaitPolicy,
11150) -> Result<Vec<CommandOutcome>> {
11151 let shell = state.config.shell();
11152 let gate_commands = state.config.verify.gate.clone();
11153 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11154 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11155 Vec::new()
11156 } else {
11157 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11158 let cache_dir = state.config.cache_dir();
11159 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11160 let (outcomes, _) = with_cache_lease(
11161 state,
11162 cache_dir.as_deref(),
11163 "gate",
11164 "gate",
11165 &winner.worktree,
11166 &head,
11167 timeout,
11168 "final gate",
11169 wait,
11170 |state, budget| {
11171 let shell = shell.clone();
11172 let gate_commands = gate_commands.clone();
11173 let winner = winner.clone();
11174 async move {
11175 Runner::run_pre_gate(state, &winner).await;
11176 let (outcomes, timed_out_pids) = run_commands(
11177 state,
11178 "gate",
11179 "gate",
11180 0,
11181 &shell,
11182 &gate_commands,
11183 &winner.worktree,
11184 budget,
11185 )
11186 .await;
11187 (outcomes, false, timed_out_pids)
11188 }
11189 },
11190 )
11191 .await?;
11192 outcomes
11193 };
11194 if outcomes.is_empty() {
11195 state.event("gate", "no gate commands configured; not tested");
11200 }
11201 for o in &outcomes {
11202 state.event(
11203 "gate",
11204 format!(
11205 "`{}` -> {}",
11206 o.command,
11207 if o.ok() {
11208 "pass".to_owned()
11209 } else {
11210 format!(
11211 "FAIL ({:?})\n{}",
11212 o.code,
11213 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11214 )
11215 }
11216 ),
11217 );
11218 }
11219 Ok(outcomes)
11220}
11221
11222pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11228 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11229 ""
11230 } else {
11231 pr_title
11232 }
11233}
11234
11235pub fn landing_subject_source(state: &RunState) -> String {
11239 if is_review_run(state) {
11240 let winner = state.candidates.first().map_or('A', |c| c.label);
11241 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11242 }
11243 state.instruction.clone()
11244}
11245
11246fn summary_without_title(summary: &str) -> String {
11249 let mut lines = summary.trim().lines().peekable();
11250 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11251 lines.next();
11252 }
11253 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11254}
11255
11256#[cfg(test)]
11268fn pr_message(state: &RunState, winner: char) -> PrMessage {
11269 pr_message_with(state, winner, None)
11270}
11271
11272#[cfg(test)]
11275fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11276 let raw = pr_message_raw(
11277 state,
11278 winner,
11279 facts,
11280 crate::pr_outcome::FinalHead::Unreported,
11281 ChangeNote::NotAsked,
11282 );
11283 let id = crate::scrub::Identity::current();
11284 PrMessage {
11285 title: crate::scrub::scrub(&raw.title, &id),
11286 body: crate::scrub::scrub(&raw.body, &id),
11287 }
11288}
11289
11290#[derive(Debug, Clone, Copy)]
11292enum ChangeNote<'a> {
11293 NotAsked,
11295 Summary(&'a crate::run::ChangeSummary),
11297 Unavailable(&'a str),
11299}
11300
11301fn pr_message_raw(
11302 state: &RunState,
11303 winner: char,
11304 facts: Option<&BranchFacts>,
11305 head: crate::pr_outcome::FinalHead<'_>,
11306 change: ChangeNote<'_>,
11307) -> PrMessage {
11308 if !matches!(change, ChangeNote::NotAsked) {
11309 let (title, mut body) = match change {
11310 ChangeNote::Summary(c) => (
11311 c.title.clone(),
11312 format!("## Summary\n\n{}\n\n", c.body.trim()),
11313 ),
11314 ChangeNote::Unavailable(why) => (
11315 neutral_title(state, winner),
11316 format!(
11317 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11318 ),
11319 ),
11320 ChangeNote::NotAsked => unreachable!(),
11321 };
11322 body.push_str(&public_pr_evidence(state, head));
11323 body.push_str(&format!(
11324 "\n---\nmagi:run/{} magi:candidate-{}\n",
11325 state.id,
11326 winner.to_ascii_lowercase()
11327 ));
11328 return PrMessage { title, body };
11329 }
11330 pr_preview(state, winner, facts, head)
11331}
11332
11333fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11336 let mut body = crate::pr_outcome::review_rounds(state);
11337 body.push_str(&crate::pr_outcome::final_verification(state, head));
11338 let open = state.last_round_findings();
11339 if !open.is_empty() {
11340 body.push_str("## Open review findings\n\n");
11341 for f in open {
11342 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11343 }
11344 body.push('\n');
11345 }
11346 if let Some(c) = state.change_summary.as_ref()
11347 && head == crate::pr_outcome::FinalHead::At(&c.head)
11348 {
11349 body.push_str(&format!(
11350 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11351 c.base, c.head
11352 ));
11353 }
11354 body
11355}
11356
11357fn pr_preview(
11359 state: &RunState,
11360 winner: char,
11361 facts: Option<&BranchFacts>,
11362 head: crate::pr_outcome::FinalHead<'_>,
11363) -> PrMessage {
11364 let summary = state
11365 .candidates
11366 .iter()
11367 .find(|c| c.label == winner)
11368 .map(|c| c.summary.as_str())
11369 .unwrap_or_default();
11370 let review = is_review_run(state);
11373 let title = if review {
11374 facts
11375 .and_then(|f| {
11376 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11377 })
11378 .or_else(|| review_title(state))
11379 .or_else(|| {
11380 state
11381 .candidates
11382 .iter()
11383 .find(|c| c.label == winner)
11384 .filter(|c| !c.branch.starts_with("magi/"))
11385 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11386 })
11387 .unwrap_or_else(|| neutral_title(state, winner))
11388 } else {
11389 summary_title(summary).unwrap_or_else(|| {
11390 english_title(
11391 &state.instruction,
11392 PR_TITLE_MAX,
11393 state.config.graph.github_english(),
11394 )
11395 .unwrap_or_else(|| neutral_title(state, winner))
11396 })
11397 };
11398
11399 let mut body = String::new();
11400 let what = summary_without_title(summary);
11401 if !what.is_empty() {
11402 body.push_str("## Summary\n\n");
11403 body.push_str(&what);
11404 body.push_str("\n\n");
11405 }
11406
11407 body.push_str(&crate::pr_outcome::review_rounds(state));
11409 body.push_str(&crate::pr_outcome::final_verification(state, head));
11410 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11411
11412 let open = state.open_findings();
11413 if !open.is_empty() {
11414 body.push_str("## Open review findings\n\n");
11415 for f in &open {
11416 body.push_str(&format!(
11417 "- `{}` [{:?}] {}\n",
11418 f.id, f.severity, "Context retained in local review records."
11419 ));
11420 }
11421 body.push('\n');
11422 }
11423
11424 if let Some(fix) = fix
11425 && !fix.rejected.is_empty()
11426 {
11427 body.push_str("## Declined by the fixer\n\n");
11428 for r in &fix.rejected {
11429 body.push_str(&format!(
11430 "- `{}`: {}\n",
11431 r.id, "Reason retained in local review records."
11432 ));
11433 }
11434 body.push('\n');
11435 }
11436
11437 if review {
11438 body.push_str("## Commits under review\n\n");
11440 if let Some(facts) = facts {
11441 let mut left = COMMITS_MAX_CHARS;
11442 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11443 let mut entry = format!("- {}\n", markdown_safe(subject));
11444 for l in markdown_safe(text).lines() {
11445 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11446 }
11447 if left == 0 {
11448 body.push_str(&format!(
11449 "- ... {} more commit(s)\n",
11450 facts.commits.len() - i
11451 ));
11452 break;
11453 }
11454 if entry.len() > left {
11455 let mut end = left;
11458 while !entry.is_char_boundary(end) {
11459 end -= 1;
11460 }
11461 entry.truncate(end);
11462 entry.push_str("\n ... (truncated)\n");
11463 left = 0;
11464 } else {
11465 left -= entry.len();
11466 }
11467 body.push_str(&entry);
11468 }
11469 if !facts.stat.trim().is_empty() {
11470 body.push_str(&format!(
11471 "\n## Diff stat\n\n```\n{}\n```\n",
11472 markdown_safe(facts.stat.trim())
11473 ));
11474 }
11475 } else {
11476 match &state.reviewed_commits {
11477 Some(subjects) => {
11478 for s in subjects {
11479 body.push_str(&format!("- {}\n", s.trim()));
11480 }
11481 }
11482 None => {
11483 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11486 body.push_str(rest.trim());
11487 body.push('\n');
11488 }
11489 }
11490 }
11491 } else {
11492 let task = state.instruction.trim();
11493 let task = if task.is_empty() {
11494 "(empty task)"
11495 } else {
11496 task
11497 };
11498 body.push_str(&format!(
11499 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11500 task.replace("</details>", "</details>")
11501 ));
11502 }
11503
11504 body.push_str(&format!(
11505 "\n---\nmagi:run/{} magi:candidate-{}\n",
11506 state.id,
11507 winner.to_ascii_lowercase()
11508 ));
11509
11510 PrMessage { title, body }
11511}
11512
11513fn seeded_instruction(state: &RunState) -> String {
11516 let mut inputs = Vec::new();
11517 if let Some(start) = &state.sources.start_from {
11518 inputs.push(format!(
11519 "Start from `{}` at `{}` (including ancestry).",
11520 start.token, start.sha
11521 ));
11522 }
11523 for pick in &state.sources.pick_commits {
11524 inputs.push(format!(
11525 "Pick `{}` at `{}` (single commit).",
11526 pick.token, pick.sha
11527 ));
11528 }
11529 if inputs.is_empty() {
11530 state.instruction.clone()
11531 } else {
11532 format!(
11533 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11534 state.instruction,
11535 inputs.join("\n")
11536 )
11537 }
11538}
11539
11540async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11544 let base = &state.base_branch;
11545 let mut against = base.clone();
11546 if mode == MergeMode::Pr {
11547 let remote = &state.config.merge.remote;
11551 let tracking = format!("{remote}/{base}");
11552 let fetched = git::fetch(repo, remote, base).await;
11553 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11554 against = tracking;
11555 } else {
11556 return false;
11557 }
11558 }
11559 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11560}
11561
11562fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11565 let mut detail = format!(
11566 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11567 nothing to open a pull request for"
11568 );
11569 detail.push_str(
11570 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11571 );
11572 detail
11573}
11574
11575#[derive(Debug, PartialEq, Eq)]
11578enum PrPlan {
11579 Create,
11580 Adopt { url: String, title: String },
11581 Stop(String),
11582}
11583
11584fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11587 match found {
11588 Ok(land::OpenPr::None) => PrPlan::Create,
11589 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11590 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11591 "several open pull requests exist for this branch, not picking one: {}",
11592 urls.join(" ")
11593 )),
11594 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11595 }
11596}
11597
11598pub(crate) async fn gh_pr_create(
11600 cwd: &Path,
11601 base: &str,
11602 head: &str,
11603 title: &str,
11604 body: &str,
11605) -> Result<String> {
11606 let out = tokio::process::Command::new("gh")
11607 .args([
11608 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11609 ])
11610 .current_dir(cwd)
11611 .quiet()
11612 .stdin(std::process::Stdio::null())
11613 .output()
11614 .await
11615 .context("spawn gh")?;
11616 if out.status.success() {
11617 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11618 } else {
11619 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11620 }
11621}
11622
11623async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11625 match std::fs::symlink_metadata(path) {
11626 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11627 git::worktree_remove_clean(repo, path).await?;
11630 return Ok(());
11631 }
11632 Err(e) => return Err(e.into()),
11633 Ok(_) => {}
11634 }
11635 if !git::is_clean(path).await? {
11636 bail!(
11637 "worktree has uncommitted work; refusing removal: {}",
11638 path.display()
11639 );
11640 }
11641 if !git::worktree_remove_clean(repo, path).await? {
11642 bail!("worktree removal refused: {}", path.display());
11643 }
11644 Ok(())
11645}
11646
11647pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11656 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11657 bail!("refusing fold: {reason}");
11658 }
11659 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11660 bail!("candidate preservation incomplete; refusing cleanup");
11661 }
11662 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11666 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11667 let _driver = crate::driver::maintenance(home, &state.id)?;
11668 let mut output_guards = Vec::new();
11669 for entry in std::fs::read_dir(state.worktree_root())
11670 .into_iter()
11671 .flatten()
11672 .flatten()
11673 {
11674 if entry.path().join(".git").is_file() {
11675 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11676 }
11677 }
11678 let repo = state.repo.clone();
11679 let wait = repository_wait(state, None);
11682 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11683 Ok(lease) => lease,
11684 Err(e) => {
11685 state.save_under(home)?;
11686 return Err(e);
11687 }
11688 };
11689 let _source_outputs = if state.status == RunStatus::Merged
11692 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11693 {
11694 verify_merged_fold_sources(state, drop_winner, home).await?
11695 } else {
11696 Vec::new()
11697 };
11698 let root = state.worktree_root();
11699 let winner = state.tally.as_ref().map(|t| t.winner);
11700 let mut removed = Vec::new();
11701
11702 for i in 0..state.candidates.len() {
11703 let c = state.candidates[i].clone();
11704 let is_winner = Some(c.label) == winner;
11705 if is_winner && !drop_winner {
11706 continue;
11707 }
11708 let existed = c.worktree.exists();
11709 remove_preserved_worktree(&repo, &c.worktree).await?;
11712 if existed {
11713 removed.push(c.worktree.to_string_lossy().into_owned());
11714 }
11715 let handed_over = state.released_branches.contains(&c.branch);
11718 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11719 if !git::branch_delete(&repo, &c.branch).await? {
11720 bail!("candidate branch removal refused");
11721 }
11722 removed.push(c.branch.clone());
11723 }
11724 state.candidates[i].folded = true;
11725 }
11726
11727 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11728 let path = name.path();
11729 let keep = !drop_winner
11730 && winner.is_some_and(|w| {
11731 path.file_name()
11732 .is_some_and(|n| n == format!("cand-{w}").as_str())
11733 });
11734 if keep {
11735 continue;
11736 }
11737 remove_preserved_worktree(&repo, &path).await?;
11738 removed.push(path.to_string_lossy().into_owned());
11739 }
11740
11741 remove_if_empty(&root);
11750
11751 if state.enabled_worktree_config && drop_winner {
11752 git::release_worktree_config(&repo).await.ok();
11756 state.enabled_worktree_config = false;
11757 }
11758 state.save_under(home)?;
11759 Ok(removed)
11760}
11761
11762async fn verify_merged_fold_sources(
11763 state: &RunState,
11764 drop_winner: bool,
11765 home: &Path,
11766) -> Result<Vec<std::fs::File>> {
11767 let winner = state.tally.as_ref().map(|t| t.winner);
11768 let mut heads = BTreeSet::new();
11769 for candidate in &state.candidates {
11770 if Some(candidate.label) == winner && !drop_winner {
11771 continue;
11772 }
11773 anyhow::ensure!(
11774 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11775 "candidate recovery source is outside the inventoried run root: {}",
11776 candidate.worktree.display()
11777 );
11778 match std::fs::symlink_metadata(&candidate.worktree) {
11779 Ok(_) => {}
11780 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11781 Err(error) => return Err(error.into()),
11782 }
11783 if !state.released_branches.contains(&candidate.branch)
11784 && git::branch_exists(&state.repo, &candidate.branch).await?
11785 {
11786 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11787 }
11788 }
11789 let mut cargo_guards = Vec::new();
11790 let entries = match std::fs::read_dir(state.worktree_root()) {
11791 Ok(entries) => Some(entries),
11792 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11793 Err(error) => return Err(error.into()),
11794 };
11795 for entry in entries.into_iter().flatten() {
11796 let path = entry?.path();
11797 let metadata = std::fs::symlink_metadata(&path)?;
11798 anyhow::ensure!(
11799 metadata.is_dir() && !metadata.file_type().is_symlink(),
11800 "non-directory or symlink recovery source retained: {}",
11801 path.display()
11802 );
11803 if !drop_winner
11804 && winner.is_some_and(|w| {
11805 path.file_name()
11806 .is_some_and(|name| name == format!("cand-{w}").as_str())
11807 })
11808 {
11809 continue;
11810 }
11811 anyhow::ensure!(
11812 path.join(".git").is_file(),
11813 "unregistered recovery source retained: {}",
11814 path.display()
11815 );
11816 anyhow::ensure!(
11817 git::git(
11818 &path,
11819 &[
11820 "status",
11821 "--porcelain",
11822 "--untracked-files=all",
11823 "--ignored",
11824 "--",
11825 ".",
11826 ":(exclude)target"
11827 ]
11828 )
11829 .await?
11830 .is_empty(),
11831 "source changes or ignored recovery files retained: {}",
11832 path.display()
11833 );
11834 match std::fs::symlink_metadata(path.join("target")) {
11835 Ok(_) => {
11836 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11837 }
11838 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11839 Err(error) => return Err(error.into()),
11840 }
11841 for checkpoint in [
11842 "rebase-merge",
11843 "rebase-apply",
11844 "sequencer",
11845 "CHERRY_PICK_HEAD",
11846 "MERGE_HEAD",
11847 "REBASE_HEAD",
11848 "REVERT_HEAD",
11849 "BISECT_LOG",
11850 "AUTO_MERGE",
11851 "MERGE_MSG",
11852 ] {
11853 let location = git::git(
11854 &path,
11855 &[
11856 "rev-parse",
11857 "--path-format=absolute",
11858 "--git-path",
11859 checkpoint,
11860 ],
11861 )
11862 .await?;
11863 anyhow::ensure!(
11864 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11865 if e.kind() == std::io::ErrorKind::NotFound),
11866 "Git recovery checkpoint retained: {checkpoint}"
11867 );
11868 }
11869 heads.insert(git::rev_parse(&path, "HEAD").await?);
11870 }
11871 crate::fold_evidence::preserve(state, home, &heads).await?;
11872
11873 Ok(cargo_guards)
11874}
11875
11876fn remove_if_empty(dir: &Path) {
11887 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11888 std::fs::remove_dir(dir).ok();
11889 }
11890}
11891
11892pub fn worst_open(state: &RunState) -> Option<Severity> {
11894 state
11895 .reviews
11896 .last()?
11897 .reviews
11898 .iter()
11899 .flat_map(|r| r.findings.iter())
11900 .map(|f| f.severity)
11901 .max()
11902}
11903
11904async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11908 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11909 git::acquire_worktree_config(repo).await?;
11910 state.enabled_worktree_config = true;
11911 }
11912 Ok(())
11913}
11914
11915#[cfg(test)]
11916mod tests {
11917 #[tokio::test]
11918 async fn launcher_base_fetch_requires_remote_success() {
11919 let tmp = tempfile::tempdir().unwrap();
11920 let repo = tmp.path().join("repo");
11921 let remote = tmp.path().join("remote");
11922 std::fs::create_dir(&repo).unwrap();
11923 std::fs::create_dir(&remote).unwrap();
11924 git::git(&remote, &["init", "--bare", "-b", "main"])
11925 .await
11926 .unwrap();
11927 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11928 git::git(
11929 &repo,
11930 &[
11931 "-c",
11932 "user.name=Fixture",
11933 "-c",
11934 "user.email=fixture@example.invalid",
11935 "commit",
11936 "--allow-empty",
11937 "-m",
11938 "fixture",
11939 ],
11940 )
11941 .await
11942 .unwrap();
11943 git::git(
11944 &repo,
11945 &["remote", "add", "origin", remote.to_str().unwrap()],
11946 )
11947 .await
11948 .unwrap();
11949 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11950 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11951 assert_eq!(
11952 resolve_base(&repo, "main", "origin", None)
11953 .await
11954 .unwrap()
11955 .commit,
11956 head
11957 );
11958 #[cfg(unix)]
11959 {
11960 let policy: crate::config::Coordination = toml::from_str(
11961 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11962 ).unwrap();
11963 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11964 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11965 let held = crate::repo_lock::SeatLease::operation_with(
11966 &repo,
11967 Some(&policy),
11968 &crate::repo_lock::WaitPolicy::default(),
11969 )
11970 .await
11971 .unwrap();
11972 assert!(held.is_some());
11973 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11974 tokio::pin!(fetch);
11975 assert!(
11976 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11977 .await
11978 .is_err()
11979 );
11980 assert!(
11981 !repo.join(".git/FETCH_HEAD").exists(),
11982 "no fetch before admission"
11983 );
11984 drop(held);
11985 let fetched = fetch.await.unwrap();
11986 assert_eq!(fetched.commit, head);
11987 assert_eq!(fetched.remote, "origin");
11988 assert_eq!(fetched.branch, "main");
11989 }
11990 git::git(
11992 &repo,
11993 &[
11994 "remote",
11995 "set-url",
11996 "origin",
11997 tmp.path().join("missing").to_str().unwrap(),
11998 ],
11999 )
12000 .await
12001 .unwrap();
12002 let error = resolve_base(&repo, "main", "origin", None)
12003 .await
12004 .unwrap_err()
12005 .to_string();
12006 assert!(error.contains("cannot read origin/main"));
12007 assert!(error.contains("refusing to branch off"));
12008 }
12009
12010 #[test]
12011 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12012 let leaked = vec!["chore(deps): update a crate".to_owned()];
12013 let own = "fix(daemon): apply a chosen action";
12014 assert!(should_retitle("", own, &leaked));
12015 assert!(should_retitle(
12016 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12017 own,
12018 &leaked
12019 ));
12020 assert!(should_retitle(
12021 "chore: land candidate A of run 1",
12022 own,
12023 &leaked
12024 ));
12025 assert!(should_retitle(
12026 "magi: candidate A (uncommitted work)",
12027 own,
12028 &leaked
12029 ));
12030 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12031 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12032 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12033 }
12034
12035 #[test]
12036 fn pr_merge_plan_creates_adopts_or_stops() {
12037 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12038 assert_eq!(
12039 pr_merge_plan(Ok(land::OpenPr::One {
12040 url: "u".into(),
12041 title: "t".into()
12042 })),
12043 PrPlan::Adopt {
12044 url: "u".into(),
12045 title: "t".into()
12046 }
12047 );
12048 let PrPlan::Stop(many) =
12049 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12050 else {
12051 panic!("many must stop");
12052 };
12053 assert!(many.contains('a') && many.contains('b'));
12054 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12055 panic!("a failed lookup must stop");
12056 };
12057 assert!(err.contains("bad token"));
12058 }
12059
12060 use super::*;
12061 use crate::run::GateStatus;
12062 use std::collections::BTreeMap;
12063 use std::time::Duration;
12064
12065 fn conductor() -> AgentSpec {
12066 AgentSpec {
12067 id: "conductor".to_owned(),
12068 kind: crate::config::AgentKind::Command,
12069 model: None,
12070 command: vec!["true".to_owned()],
12071 extra_args: Vec::new(),
12072 env: BTreeMap::new(),
12073 prompt_delivery: None,
12074 max_turn_secs: None,
12075 }
12076 }
12077
12078 fn spec(id: &str) -> AgentSpec {
12079 AgentSpec {
12080 id: id.to_owned(),
12081 kind: crate::config::AgentKind::Command,
12082 model: None,
12083 command: vec!["true".to_owned()],
12084 extra_args: Vec::new(),
12085 env: BTreeMap::new(),
12086 prompt_delivery: None,
12087 max_turn_secs: None,
12088 }
12089 }
12090
12091 fn ids(xs: &[&str]) -> BTreeSet<String> {
12092 xs.iter().map(|s| (*s).to_owned()).collect()
12093 }
12094
12095 #[test]
12096 fn pick_successor_skips_an_agent_another_seat_holds() {
12097 let roster = [spec("a"), spec("b"), spec("c")];
12099 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12100 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12101 }
12102
12103 #[test]
12104 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12105 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12107 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12108 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12109 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12110 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12111 }
12112
12113 #[test]
12114 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12115 let roster = [spec("a"), spec("b")];
12116 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12117 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12118 let carried = ids(&["b"]);
12119 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12120 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12121 }
12122
12123 #[test]
12124 fn pick_successor_returns_none_without_any_untried_successor() {
12125 let roster = [spec("a"), spec("b")];
12126 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12127 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12128 let carried = ids(&["a"]);
12129 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12130 }
12131
12132 #[test]
12133 fn pick_successor_rescue_avoids_another_seats_occupant() {
12134 let roster = [spec("a"), spec("b"), spec("c")];
12136 let carried = ids(&["b", "c"]);
12137 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12138 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12139 }
12140
12141 #[test]
12142 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12143 let roster = [spec("a"), spec("b"), spec("c")];
12144 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12145 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12146 }
12147
12148 #[test]
12149 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12150 let roster = [spec("a"), spec("b"), spec("c")];
12151 let failed = ids(&["a", "b", "c"]);
12152 let mut tried = ids(&["b"]);
12154 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12155 assert_eq!(first.id, "a");
12156 tried.insert(first.id.clone());
12157 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12158 assert_eq!(second.id, "c");
12159 tried.insert(second.id.clone());
12160 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12161 }
12162
12163 #[test]
12164 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12165 let roster = [spec("a"), spec("b")];
12166 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12167 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12168 }
12169
12170 #[test]
12171 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12172 let roster = [spec("a"), spec("b"), spec("c")];
12173 let h = SeatHistory {
12174 failed: ids(&["a"]),
12175 last_ok: Some("b".to_owned()),
12176 last_fail: None,
12177 };
12178 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12179 let gone = SeatHistory {
12181 last_ok: Some("zzz".to_owned()),
12182 ..h.clone()
12183 };
12184 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12185 let failed = SeatHistory {
12186 failed: ids(&["a", "b"]),
12187 last_ok: Some("b".to_owned()),
12188 last_fail: None,
12189 };
12190 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12191 }
12192
12193 #[test]
12194 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12195 let first = SeatState::new("review-1", "alpha", 7);
12196 let next = handover_seat("review-1", "gamma", 7);
12197 assert_ne!(first.claude_session, next.claude_session);
12198 }
12199
12200 #[test]
12201 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12202 let mut state = state_with_summary("x", "y");
12203 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12204 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12205 assert_ne!(a.claude_session, b.claude_session);
12206 }
12207
12208 #[test]
12209 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12210 let roster = [spec("a"), spec("b")];
12211 let h = SeatHistory {
12212 failed: ids(&["a", "b"]),
12213 ..SeatHistory::default()
12214 };
12215 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12216 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12217 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12218 }
12219
12220 #[test]
12227 fn failure_signature_ignores_numbers_and_paths() {
12228 assert_eq!(
12229 failure_signature("exited with Some(2) and no usable output"),
12230 failure_signature("exited with Some(137) and no usable output")
12231 );
12232 assert_eq!(
12233 failure_signature("process exited with no usable output (exit code 2)"),
12234 failure_signature("process exited with no usable output (exit code 137)")
12235 );
12236 for (old, new) in [
12237 (
12238 "exited with Some(2) and no usable output",
12239 "process exited with no usable output (exit code 2)",
12240 ),
12241 (
12242 "exited with None and no usable output",
12243 "process exited with no usable output (exit code unavailable)",
12244 ),
12245 ] {
12246 let previous = FailClass::Other(failure_signature(old));
12247 let current = FailClass::Other(failure_signature(new));
12248 assert_eq!(previous, current, "resumption retains the failure class");
12249 assert!(!should_hand_over(Some(&previous), ¤t));
12250 }
12251 assert_eq!(
12252 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12253 failure_signature("cannot open /var/x.txt: denied")
12254 );
12255 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12256 }
12257
12258 #[test]
12259 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12260 let other = FailClass::Other("x".into());
12261 assert!(should_hand_over(None, &FailClass::Quota));
12262 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12263 assert!(should_hand_over(
12264 Some(&FailClass::Timeout),
12265 &FailClass::Timeout
12266 ));
12267 assert!(should_hand_over(None, &other));
12268 assert!(!should_hand_over(Some(&other), &other));
12269 assert!(should_hand_over(
12270 Some(&other),
12271 &FailClass::Other("y".into())
12272 ));
12273 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12275 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12276 }
12277
12278 #[test]
12279 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12280 let a = SeatState::new("judge-1", "alpha", 7);
12281 let b = handover_seat("judge-1", "beta", 7);
12282 assert_ne!(a.claude_session, b.claude_session);
12283 assert_eq!(b.turns, 0);
12284 }
12285
12286 #[test]
12287 fn a_timeout_is_classified_apart_from_other_failures() {
12288 assert_eq!(
12289 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12290 Some(FailClass::Timeout)
12291 );
12292 assert!(matches!(
12293 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12294 Some(FailClass::Other(_))
12295 ));
12296 }
12297
12298 #[test]
12299 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12300 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12301 let tried = BTreeSet::from(["beta".to_owned()]);
12302 let next = next_untried_in_roster(&roster, 1, &tried);
12305 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12306 }
12307
12308 #[test]
12309 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12310 let roster = vec![spec("alpha"), spec("beta")];
12311 let tried = BTreeSet::from(["beta".to_owned()]);
12312 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12316 }
12317
12318 #[test]
12319 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12320 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12321 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12322 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12326 }
12327
12328 #[test]
12329 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12330 let roster = vec![spec("a"), spec("a"), spec("b")];
12331 let tried = BTreeSet::from(["a".to_owned()]);
12332 let next = next_untried_in_roster(&roster, 0, &tried);
12333 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12334 }
12335
12336 #[test]
12337 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12338 let roster = vec![spec("a"), spec("b")];
12339 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12340 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12341 }
12342
12343 #[test]
12344 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12345 let dir = tempfile::tempdir().unwrap();
12346 let bay = dir.path().join("ffff");
12347
12348 remove_if_empty(&bay);
12350 assert!(!bay.exists());
12351
12352 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12355 remove_if_empty(&bay);
12356 assert!(bay.exists(), "non-empty directory must survive");
12357
12358 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12360 remove_if_empty(&bay);
12361 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12362 }
12363
12364 #[test]
12373 fn a_full_panel_that_found_nothing_is_clean() {
12374 assert!(round_is_clean(
12375 0,
12376 true,
12377 2,
12378 2,
12379 0,
12380 IncompleteReviewPolicy::Block
12381 ));
12382 }
12383
12384 #[test]
12385 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12386 assert!(!round_is_clean(
12387 0,
12388 true,
12389 1,
12390 2,
12391 0,
12392 IncompleteReviewPolicy::Block
12393 ));
12394 }
12395
12396 #[test]
12397 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12398 assert!(!round_is_clean(
12399 1,
12400 true,
12401 1,
12402 2,
12403 0,
12404 IncompleteReviewPolicy::Warn
12405 ));
12406 }
12407
12408 #[test]
12409 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12410 assert!(round_is_clean(
12411 0,
12412 true,
12413 1,
12414 2,
12415 0,
12416 IncompleteReviewPolicy::Warn
12417 ));
12418 }
12419
12420 #[test]
12421 fn a_full_panel_with_an_open_finding_is_not_clean() {
12422 assert!(!round_is_clean(
12423 1,
12424 true,
12425 2,
12426 2,
12427 0,
12428 IncompleteReviewPolicy::Block
12429 ));
12430 }
12431
12432 #[test]
12433 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12434 assert!(!round_is_clean(
12435 0,
12436 false,
12437 2,
12438 2,
12439 0,
12440 IncompleteReviewPolicy::Block
12441 ));
12442 }
12443
12444 #[test]
12451 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12452 assert!(round_is_clean(
12455 0,
12456 true,
12457 1,
12458 2,
12459 1,
12460 IncompleteReviewPolicy::Block
12461 ));
12462 }
12463
12464 #[test]
12465 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12466 assert!(!round_is_clean(
12469 0,
12470 true,
12471 1,
12472 2,
12473 0,
12474 IncompleteReviewPolicy::Block
12475 ));
12476 }
12477
12478 #[test]
12479 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12480 assert!(!round_is_clean(
12481 1,
12482 true,
12483 1,
12484 2,
12485 1,
12486 IncompleteReviewPolicy::Block
12487 ));
12488 assert!(!round_is_clean(
12489 0,
12490 false,
12491 1,
12492 2,
12493 1,
12494 IncompleteReviewPolicy::Block
12495 ));
12496 }
12497
12498 #[test]
12499 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12500 assert!(!round_is_clean(
12504 0,
12505 true,
12506 0,
12507 2,
12508 2,
12509 IncompleteReviewPolicy::Block
12510 ));
12511 }
12512
12513 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12514 CommandOutcome {
12515 command: "test".to_owned(),
12516 code,
12517 output_tail: String::new(),
12518 duration_ms: 0,
12519 resource_blocked,
12520 }
12521 }
12522
12523 #[test]
12524 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12525 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12526 assert!(
12527 !verify_inconclusive(&[outcome(Some(1), false)]),
12528 "an ordinary failure is still evidence about the patch"
12529 );
12530 assert!(verify_inconclusive(&[outcome(None, true)]));
12531 assert!(
12532 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12533 "one inconclusive outcome taints the whole batch"
12534 );
12535 assert!(!verify_inconclusive(&[]));
12536 }
12537
12538 #[tokio::test]
12539 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12540 let calls = std::sync::atomic::AtomicUsize::new(0);
12544 let started = Instant::now();
12545 wait_for_pids_with(
12546 &[123],
12547 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12548 Duration::from_millis(5),
12549 Duration::from_secs(5),
12550 )
12551 .await;
12552 assert!(
12553 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12554 "must keep checking rather than deciding on the first answer"
12555 );
12556 assert!(
12557 started.elapsed() < Duration::from_secs(1),
12558 "must return the moment it is confirmed dead, not wait out the ceiling"
12559 );
12560 }
12561
12562 #[tokio::test]
12563 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12564 let started = Instant::now();
12565 wait_for_pids_with(
12566 &[123],
12567 |_| true, Duration::from_millis(5),
12569 Duration::from_millis(30),
12570 )
12571 .await;
12572 let elapsed = started.elapsed();
12573 assert!(
12574 elapsed >= Duration::from_millis(30),
12575 "must not give up before its own ceiling: {elapsed:?}"
12576 );
12577 assert!(
12578 elapsed < Duration::from_secs(1),
12579 "must not wait past its own ceiling either: {elapsed:?}"
12580 );
12581 }
12582
12583 #[tokio::test]
12584 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12585 let started = Instant::now();
12586 wait_for_pids_with(
12587 &[],
12588 |_| true,
12589 Duration::from_secs(5),
12590 Duration::from_secs(5),
12591 )
12592 .await;
12593 assert!(
12594 started.elapsed() < Duration::from_millis(200),
12595 "an empty pid list has nothing to confirm"
12596 );
12597 }
12598
12599 fn review_round(
12605 clean: bool,
12606 blocking: usize,
12607 answered: usize,
12608 expected: usize,
12609 progressed: bool,
12610 e2e_ok: bool,
12611 ) -> ReviewRound {
12612 ReviewRound {
12613 round: 1,
12614 head: "h".to_owned(),
12615 verified_head: None,
12616 verified_at: None,
12617 reviews: Vec::new(),
12618 e2e: vec![CommandOutcome {
12619 command: "test".to_owned(),
12620 code: Some(if e2e_ok { 0 } else { 1 }),
12621 output_tail: String::new(),
12622 duration_ms: 0,
12623 resource_blocked: false,
12624 }],
12625 verify_retried: false,
12626 e2e_deferred: false,
12627 e2e_defer_reason: None,
12628 fix: None,
12629 blocking,
12630 answered,
12631 expected,
12632 clean,
12633 progressed,
12634 vote_split: false,
12635 reconsideration: Vec::new(),
12636 verdict: None,
12637 }
12638 }
12639
12640 #[test]
12641 fn review_conclusion_is_none_when_nothing_has_run() {
12642 assert_eq!(review_conclusion(&[], 3), None);
12643 }
12644
12645 #[test]
12646 fn review_conclusion_is_none_while_rounds_remain() {
12647 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12648 assert_eq!(review_conclusion(&rounds, 3), None);
12649 }
12650
12651 #[test]
12652 fn review_conclusion_is_gating_once_a_round_is_clean() {
12653 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12654 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12655 }
12656
12657 #[test]
12658 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12659 let rounds = vec![
12660 review_round(false, 1, 2, 2, true, true),
12661 review_round(false, 1, 2, 2, true, true),
12662 ];
12663 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12664 }
12665
12666 #[test]
12667 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12668 let rounds = vec![
12669 review_round(false, 1, 2, 2, true, true),
12670 review_round(false, 1, 2, 2, true, false),
12671 ];
12672 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12673 }
12674
12675 #[test]
12676 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12677 let mut blocked = review_round(false, 1, 2, 2, true, false);
12684 blocked.e2e[0].resource_blocked = true;
12685 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12686 assert_eq!(review_conclusion(&rounds, 2), None);
12687 }
12688
12689 #[test]
12690 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12691 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12693 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12694 }
12695
12696 #[test]
12697 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12698 let rounds = vec![
12699 review_round(false, 1, 2, 2, false, true),
12700 review_round(false, 1, 2, 2, false, true),
12701 ];
12702 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12703 }
12704
12705 fn secs(n: u64) -> Duration {
12706 Duration::from_secs(n)
12707 }
12708
12709 #[tokio::test]
12712 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12713 ask_test_home();
12714 let tmp = tempfile::tempdir().unwrap();
12715 let repo = tmp.path().join("repo");
12716 std::fs::create_dir_all(&repo).unwrap();
12717 init_repo(&repo);
12718 let g = |args: &[&str]| {
12719 let out = std::process::Command::new("git")
12720 .args(args)
12721 .current_dir(&repo)
12722 .quiet()
12723 .output()
12724 .unwrap();
12725 assert!(
12726 out.status.success(),
12727 "{}",
12728 String::from_utf8_lossy(&out.stderr)
12729 );
12730 };
12731 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12732 g(&["checkout", "-b", "historical"]);
12733 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12734 g(&["add", "."]);
12735 g(&["commit", "-m", "old alternative"]);
12736 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12737 g(&["checkout", "-b", "magi/task-selected", &base]);
12738 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12739 g(&["add", "."]);
12740 g(&["commit", "-m", "measurement infrastructure"]);
12741 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12742 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12743 g(&["add", "."]);
12744 g(&["commit", "-m", "dependent implementation"]);
12745 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12746 g(&["checkout", "main"]);
12747 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12748 g(&["add", "."]);
12749 g(&["commit", "-m", "unrelated base change"]);
12750 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12751 let origin = tmp.path().join("origin.git");
12752 g(&[
12753 "clone",
12754 "--bare",
12755 repo.to_str().unwrap(),
12756 origin.to_str().unwrap(),
12757 ]);
12758 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12759 g(&["fetch", "origin"]);
12760 let selection = crate::pr_recovery::Selection {
12761 url: "https://example.test/project/pull/1".into(),
12762 branch: "magi/task-selected".into(),
12763 head: head.clone(),
12764 source_head: None,
12765 source_run: None,
12766 source_proof: None,
12767 disposition: Some(crate::pr_recovery::Disposition::Review),
12768 instruction: "preserve measured behavior".into(),
12769 approved_at: jiff::Timestamp::now(),
12770 };
12771 let mut template = runner_at(RunStatus::Reviewing);
12772 template.state.repo = repo.clone();
12773 template.state.base_commit = current.clone();
12774 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12775 template.state.origin = Some(crate::run::Origin::queue("selected"));
12776 template.state.pr_head = Some(selection.branch.clone());
12777 template.state.pr_recovery = Some(selection.clone());
12778 let mut runner =
12779 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12780 .await
12781 .unwrap();
12782 let wt = runner.state.winner().unwrap().worktree.clone();
12783 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12784 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12785 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12786 assert!(runner.state.seeds.is_empty());
12787 assert_eq!(runner.state.instruction, "preserve measured behavior");
12788 runner.sync_to_base().await.unwrap();
12789 assert_eq!(
12790 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12791 "prerequisite"
12792 );
12793 assert_eq!(
12794 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12795 "measured implementation"
12796 );
12797 assert_eq!(
12798 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12799 "current main change"
12800 );
12801 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12802 assert_eq!(
12803 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12804 head
12805 );
12806
12807 let mut partial = runner_at(RunStatus::Reviewing);
12809 partial.state.repo = repo.clone();
12810 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12811 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12812 run: Some("exit 1".into()),
12813 ..Default::default()
12814 }];
12815 let selection = runner.state.pr_recovery.clone().unwrap();
12816 partial.state.pr_recovery = Some(selection.clone());
12817 partial.state.pr_head = Some(selection.branch.clone());
12818 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12819 let id = partial.state.id.clone();
12820 let error = match Runner::open_selected(
12821 &repo,
12822 partial.state,
12823 partial.roles,
12824 1,
12825 runner.state.base_commit.clone(),
12826 selection,
12827 )
12828 .await
12829 {
12830 Ok(_) => panic!("setup must fail"),
12831 Err(e) => e,
12832 };
12833 assert_eq!(
12834 error
12835 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12836 .unwrap()
12837 .run,
12838 id
12839 );
12840 let failed = RunState::load(&id).unwrap();
12841 assert_eq!(failed.status, RunStatus::Blocked);
12842 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12843 assert!(Runner::resume(&id).is_err());
12844 runner.state.status = RunStatus::Ready;
12845 runner.state.driver_exited = true;
12846 runner.state.save().unwrap();
12847 let mut retry = runner_at(RunStatus::Reviewing);
12848 retry.state.repo = repo.clone();
12849 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12850 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12851 reselected.approved_at = jiff::Timestamp::now();
12852 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12853 retry.state.pr_head = Some(reselected.branch.clone());
12854 retry.state.pr_recovery = Some(reselected.clone());
12855 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12856 .await
12857 .unwrap();
12858 let restarted = Runner::open_selected(
12859 &repo,
12860 retry.state,
12861 retry.roles,
12862 1,
12863 runner.state.base_commit.clone(),
12864 reselected,
12865 )
12866 .await
12867 .unwrap();
12868 assert_eq!(
12869 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12870 .await
12871 .unwrap(),
12872 head
12873 );
12874 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12875 assert_eq!(
12876 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12877 head
12878 );
12879 }
12880
12881 #[tokio::test]
12882 #[cfg(unix)]
12883 async fn fallback_and_interrupted_restoration_require_coordination() {
12884 use std::os::fd::AsRawFd as _;
12885 ask_test_home();
12886 let tmp = tempfile::tempdir().unwrap();
12887 let mut runner = registered_runner(tmp.path()).await;
12888 runner.state.config.graph.timeout_resource_wait = 1;
12889 let winner = runner.state.candidates[0].clone();
12890 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12891 let leftover = winner.worktree.join("leftover");
12892 std::fs::write(&leftover, "untrusted edits").unwrap();
12893 let next = tmp.path().join("next-fixer");
12894 let mut failed = spec("first");
12895 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12896 let mut successor = spec("second");
12897 successor.command = vec![
12898 "sh".into(),
12899 "-c".into(),
12900 format!("touch '{}'; echo done", next.display()),
12901 ];
12902 let attempts = vec![
12903 (failed.clone(), "first".into()),
12904 (successor.clone(), "second".into()),
12905 ];
12906 let lock_path = runner.state.repo.join(".git/updater.lock");
12907 let external = std::fs::File::create(&lock_path).unwrap();
12908 assert_eq!(
12909 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12910 0
12911 );
12912 let artifacts = tmp.path().join("artifacts");
12913 let build = |spec, seat| SeatJob {
12914 spec,
12915 seat,
12916 cwd: winner.worktree.clone(),
12917 prompt: "fixture".into(),
12918 timeout: Duration::from_secs(10),
12919 allow_write: true,
12920 sessions: false,
12921 artifacts: artifacts.clone(),
12922 stem: "fix".into(),
12923 handover: None,
12924 };
12925 let (_, _, out) = runner
12926 .ask_fixer_among(
12927 attempts.clone(),
12928 "graft",
12929 None,
12930 Some((&winner.worktree, &head)),
12931 build,
12932 )
12933 .await;
12934 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12935 assert!(leftover.exists(), "contention must prevent clean");
12936 assert!(!next.exists());
12937 assert!(runner.state.handovers.is_empty());
12938 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12940 from_run: "earlier".into(),
12941 generation: 1,
12942 mode: crate::recovery::Mode::VerifyRepair,
12943 reasons: Vec::new(),
12944 reviewed_head: None,
12945 rounds: Vec::new(),
12946 open_findings: Vec::new(),
12947 evidence: None,
12948 criteria_inherited: true,
12949 repair: crate::recovery::Repair::Started {
12950 before: head.clone(),
12951 },
12952 });
12953 assert!(
12954 runner
12955 .recovery_repair(&winner)
12956 .await
12957 .unwrap_err()
12958 .is::<crate::repo_lock::ResourceBlocked>()
12959 );
12960 assert!(matches!(
12961 runner.state.recovery.as_ref().unwrap().repair,
12962 crate::recovery::Repair::Started { .. }
12963 ));
12964 assert!(leftover.exists());
12965 drop(external);
12966 runner.recovery_repair(&winner).await.unwrap();
12967 assert!(!leftover.exists());
12968 assert!(matches!(
12969 runner.state.recovery.as_ref().unwrap().repair,
12970 crate::recovery::Repair::Done { .. }
12971 ));
12972 std::fs::remove_file(&lock_path).unwrap();
12974 std::fs::create_dir(&lock_path).unwrap();
12975 std::fs::write(&leftover, "untrusted edits").unwrap();
12976 let (_, _, out) = runner
12977 .ask_fixer_among(
12978 attempts,
12979 "graft",
12980 None,
12981 Some((&winner.worktree, &head)),
12982 build,
12983 )
12984 .await;
12985 assert!(matches!(out, AgentOutcome::Failed(_)));
12986 assert!(leftover.exists());
12987 assert!(!next.exists());
12988 assert_eq!(
12989 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12990 head
12991 );
12992 }
12993
12994 #[cfg(unix)]
12995 async fn registered_runner(root: &Path) -> Runner {
12996 ask_test_home();
12997 let repo = root.join("repo");
12998 std::fs::create_dir_all(&repo).unwrap();
12999 init_repo(&repo);
13000 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13001 let mut runner = runner_at(RunStatus::Prep);
13002 runner.state.repo = repo.clone();
13003 runner.state.config.coordination = crate::config::test_coordination();
13004 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13005 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13006 runner.state.config.graph.advise = false;
13007 runner.roles.implementers = vec![spec("implement")];
13008 runner.roles.judges = vec![spec("judge")];
13009 runner.prep().await.unwrap();
13010 runner
13011 }
13012
13013 #[tokio::test]
13014 #[cfg(unix)]
13015 async fn parked_implementer_persists_completed_work() {
13016 for registered in [false, true] {
13017 let tmp = tempfile::tempdir().unwrap();
13018 let mut runner = registered_runner(tmp.path()).await;
13019 if !registered {
13020 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13021 }
13022 let marker = tmp.path().join("started");
13023 let release = tmp.path().join("release");
13024 let mut implementer = spec("implement");
13025 implementer.command = vec![
13026 "sh".into(),
13027 "-c".into(),
13028 format!(
13029 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13030 marker.display(),
13031 release.display()
13032 ),
13033 ];
13034 runner.roles.implementers = vec![implementer.clone()];
13035 runner.state.config.agents = vec![implementer];
13036 let pause = runner.pause.clone();
13037 let (result, ()) = tokio::join!(runner.implement(), async {
13038 tokio::time::timeout(Duration::from_secs(10), async {
13039 while !marker.exists() {
13040 tokio::time::sleep(Duration::from_millis(10)).await;
13041 }
13042 })
13043 .await
13044 .unwrap();
13045 pause.park();
13046 std::fs::write(&release, "go").unwrap();
13047 });
13048 result.unwrap();
13049 let candidate = &runner.state.candidates[0];
13050 assert!(candidate.commits > 0, "{candidate:?}");
13051 assert!(candidate.summary.contains("saved-result"));
13052 assert_eq!(candidate.files, 1);
13053 let before = candidate.commits;
13054 runner.implement().await.unwrap();
13055 assert_eq!(runner.state.candidates[0].commits, before);
13056 }
13057 }
13058
13059 #[tokio::test]
13060 #[cfg(unix)]
13061 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13062 let tmp = tempfile::tempdir().unwrap();
13063 let mut runner = registered_runner(tmp.path()).await;
13064 let winner = runner.state.candidates[0].clone();
13065 let cache = tmp.path().join("cache");
13066 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13067 runner.state.config.graph.timeout_resource_wait = 0;
13068 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13069 let crate::cache::AcquireOutcome::Acquired(held) =
13070 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13071 else {
13072 panic!("cache must be free")
13073 };
13074 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13075 from_run: "earlier".into(),
13076 generation: 1,
13077 mode: crate::recovery::Mode::VerifyRepair,
13078 reasons: Vec::new(),
13079 reviewed_head: None,
13080 rounds: Vec::new(),
13081 open_findings: Vec::new(),
13082 evidence: None,
13083 criteria_inherited: true,
13084 repair: crate::recovery::Repair::NotStarted,
13085 });
13086 runner.recovery_repair(&winner).await.unwrap();
13087 assert!(matches!(
13088 runner.state.recovery.as_ref().unwrap().repair,
13089 crate::recovery::Repair::WaitingForRestore { .. }
13090 ));
13091 drop(held);
13092 runner.state.resource_block = None;
13093 runner.state.config.graph.timeout_resource_wait = 1;
13094 runner.recovery_repair(&winner).await.unwrap();
13095 assert!(matches!(
13096 runner.state.recovery.as_ref().unwrap().repair,
13097 crate::recovery::Repair::Done { .. }
13098 ));
13099 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13100 }
13101
13102 #[tokio::test]
13103 #[cfg(unix)]
13104 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13105 use std::os::fd::AsRawFd as _;
13106 for graft in [false, true] {
13107 let tmp = tempfile::tempdir().unwrap();
13108 let mut runner = registered_runner(tmp.path()).await;
13109 runner.state.config.graph.timeout_resource_wait = 1;
13110 let winner = runner.state.candidates[0].clone();
13111 let marker = tmp.path().join("fixer-started");
13112 let mut fixer = spec("fixer");
13113 fixer.command = vec![
13114 "sh".into(),
13115 "-c".into(),
13116 format!("echo started >> '{}'; echo done", marker.display()),
13117 ];
13118 runner.roles.fixer = Some(vec![fixer]);
13119 if graft {
13120 let mut loser = winner.clone();
13121 loser.label = 'B';
13122 loser.index = 1;
13123 runner.state.candidates.push(loser);
13124 runner.state.config.graph.graft_runner_up = true;
13125 runner.state.tally = Some(Tally {
13126 first_choice: BTreeMap::new(),
13127 borda: BTreeMap::new(),
13128 winner: winner.label,
13129 rankings: 1,
13130 unanimous_initial: true,
13131 deliberated: false,
13132 changed_votes: 0,
13133 unanimous_final: true,
13134 tie_break: None,
13135 judges: 1,
13136 present: 1,
13137 quorum: 1,
13138 met_quorum: true,
13139 uncontested: None,
13140 });
13141 runner.state.judgements.push(crate::run::Judgement {
13142 judge: 1,
13143 seat: "judge-1".into(),
13144 agent: "judge".into(),
13145 ranking: vec![winner.label, 'B'],
13146 reasons: BTreeMap::new(),
13147 confidence: None,
13148 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13149 order: vec![0, 1],
13150 failed: None,
13151 duration_ms: 0,
13152 });
13153 } else {
13154 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13155 from_run: "earlier".into(),
13156 generation: 1,
13157 mode: crate::recovery::Mode::VerifyRepair,
13158 reasons: Vec::new(),
13159 reviewed_head: None,
13160 rounds: Vec::new(),
13161 open_findings: Vec::new(),
13162 evidence: None,
13163 criteria_inherited: true,
13164 repair: crate::recovery::Repair::NotStarted,
13165 });
13166 }
13167 let reader = crate::repo_lock::SeatLease::for_checkout(
13168 &winner.worktree,
13169 false,
13170 crate::config::test_coordination().as_ref(),
13171 )
13172 .await
13173 .unwrap();
13174 let pause = runner.pause.clone();
13175 let (result, ()) = tokio::join!(
13176 async {
13177 if graft {
13178 runner.graft().await
13179 } else {
13180 runner.recovery_repair(&winner).await
13181 }
13182 },
13183 async {
13184 tokio::time::sleep(Duration::from_millis(100)).await;
13185 pause.park();
13186 }
13187 );
13188 result.unwrap();
13189 assert!(!marker.exists(), "a blocked seat must not spawn");
13190 runner.state = RunState::load(&runner.state.id).unwrap();
13192 if graft {
13193 assert_eq!(
13194 runner.state.graft.as_ref().unwrap().outcome,
13195 GraftOutcome::WaitingForRestore
13196 );
13197 } else {
13198 assert!(matches!(
13199 runner.state.recovery.as_ref().unwrap().repair,
13200 crate::recovery::Repair::WaitingForRestore { .. }
13201 ));
13202 }
13203 drop(reader);
13204 runner.pause = Pause::new();
13205 runner.state.resource_block = None;
13206 let external =
13208 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13209 assert_eq!(
13210 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13211 0
13212 );
13213 let leftover = winner.worktree.join("leftover");
13214 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13215 let result = if graft {
13216 runner.graft().await
13217 } else {
13218 runner.recovery_repair(&winner).await
13219 };
13220 assert!(
13221 result
13222 .unwrap_err()
13223 .is::<crate::repo_lock::ResourceBlocked>()
13224 );
13225 assert!(leftover.exists());
13226 assert!(!marker.exists());
13227 drop(external);
13228 runner.state.resource_block = None;
13229 let result = if graft {
13230 runner.graft().await
13231 } else {
13232 runner.recovery_repair(&winner).await
13233 };
13234 result.unwrap();
13235 assert!(!leftover.exists(), "restore must precede retry");
13236 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13237 if graft {
13238 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13239 } else {
13240 assert!(matches!(
13241 runner.state.recovery.as_ref().unwrap().repair,
13242 crate::recovery::Repair::Done { .. }
13243 ));
13244 }
13245 }
13246 }
13247
13248 #[tokio::test]
13249 #[cfg(unix)]
13250 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13251 use std::os::fd::AsRawFd as _;
13252 let tmp = tempfile::tempdir().unwrap();
13253 let mut runner = registered_runner(tmp.path()).await;
13254 runner.state.config.graph.timeout_resource_wait = 1;
13255 let winner = runner.state.candidates[0].clone();
13256 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13257 runner.state.tally = Some(Tally {
13258 first_choice: BTreeMap::new(),
13259 borda: BTreeMap::new(),
13260 winner: winner.label,
13261 rankings: 0,
13262 unanimous_initial: false,
13263 deliberated: false,
13264 changed_votes: 0,
13265 unanimous_final: false,
13266 tie_break: None,
13267 judges: 0,
13268 present: 0,
13269 quorum: 0,
13270 met_quorum: true,
13271 uncontested: Some("fixture".into()),
13272 });
13273 runner.state.graft = Some(GraftRecord {
13274 pre: pre.clone(),
13275 post: None,
13276 lines: None,
13277 limit: None,
13278 agent: None,
13279 outcome: GraftOutcome::Running,
13280 requested: Vec::new(),
13281 adoption: None,
13282 adoption_error: None,
13283 });
13284 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13285 git::commit_all(&winner.worktree, "unfinished graft")
13286 .await
13287 .unwrap();
13288 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13289 let leftover = winner.worktree.join("leftover");
13290 std::fs::write(&leftover, "untrusted edits").unwrap();
13291 let path = runner.state.repo.join(".git/updater.lock");
13292 let external = std::fs::File::create(&path).unwrap();
13293 assert_eq!(
13294 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13295 0
13296 );
13297 assert!(
13298 runner
13299 .graft()
13300 .await
13301 .unwrap_err()
13302 .is::<crate::repo_lock::ResourceBlocked>()
13303 );
13304 assert!(matches!(
13305 runner.state.graft.as_ref().unwrap().outcome,
13306 GraftOutcome::Running
13307 ));
13308 assert_eq!(
13309 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13310 unfinished
13311 );
13312 assert!(leftover.exists());
13313 drop(external);
13314 std::fs::remove_file(&path).unwrap();
13315 std::fs::create_dir(&path).unwrap();
13316 assert!(runner.graft().await.is_err());
13317 assert!(matches!(
13318 runner.state.graft.as_ref().unwrap().outcome,
13319 GraftOutcome::Running
13320 ));
13321 assert_eq!(
13322 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13323 unfinished
13324 );
13325 assert!(leftover.exists());
13326 std::fs::remove_dir(&path).unwrap();
13327 runner.graft().await.unwrap();
13328 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13329 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13330 assert!(!leftover.exists());
13331 }
13332
13333 #[tokio::test]
13334 #[cfg(unix)]
13335 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13336 use std::os::fd::AsRawFd as _;
13337 let tmp = tempfile::tempdir().unwrap();
13338 let mut runner = registered_runner(tmp.path()).await;
13339 runner.state.prep_complete = Some(false);
13340 runner.state.config.graph.timeout_resource_wait = 1;
13341 runner.state.save().unwrap();
13342 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13343 assert_eq!(
13344 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13345 0
13346 );
13347 runner.execute().await.unwrap();
13348 let disk = RunState::load(&runner.state.id).unwrap();
13349 assert!(disk.driver_exited);
13350 assert_eq!(disk.driver_pid, None);
13351 assert_eq!(runner.state.status, RunStatus::Blocked);
13352 assert_eq!(
13353 runner.state.resource_block.as_ref().unwrap().kind,
13354 "repository coordination"
13355 );
13356 assert!(runner.state.prep_failure.is_none());
13357 assert!(runner.state.seats.is_empty());
13358 runner.pause.park();
13359 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13362 .await
13363 .unwrap()
13364 .unwrap();
13365 assert!(runner.state.parked);
13366 assert!(runner.state.prep_failure.is_none());
13367 let disk = RunState::load(&runner.state.id).unwrap();
13368 assert!(disk.driver_exited);
13369 assert_eq!(disk.driver_pid, None);
13370 }
13371
13372 #[tokio::test]
13373 #[cfg(unix)]
13374 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13375 use std::os::fd::AsRawFd as _;
13376 let tmp = tempfile::tempdir().unwrap();
13377 let mut runner = registered_runner(tmp.path()).await;
13378 runner.state.prep_complete = Some(false);
13379 runner.state.config.graph.timeout_resource_wait = 30;
13380 runner.state.save().unwrap();
13381 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13382 assert_eq!(
13383 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13384 0
13385 );
13386 assert!(
13387 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13388 .await
13389 .is_err()
13390 );
13391 let disk = RunState::load(&runner.state.id).unwrap();
13392 assert!(disk.driver_exited);
13393 assert_eq!(disk.driver_pid, None);
13394 assert!(disk.status.resumable());
13395 assert!(runner.state.driver_exited);
13396 assert_eq!(runner.state.driver_pid, None);
13397 runner.state.save().unwrap();
13398 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13399 }
13400
13401 #[tokio::test]
13402 #[cfg(unix)]
13403 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13404 let tmp = tempfile::tempdir().unwrap();
13405 let mut runner = registered_runner(tmp.path()).await;
13406 let winner = runner.state.candidates[0].clone();
13407 let cache = tmp.path().join("cache");
13408 let started = tmp.path().join("pre-gate-started");
13409 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13410 runner.state.config.verify.gate =
13411 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13412 let home = crate::run::home();
13413 let owner =
13414 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13415 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13416 crate::cache::AcquireOutcome::Acquired(g) => g,
13417 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13418 };
13419 let mut gate = Box::pin(runner.run_gate(&winner));
13420 assert!(
13421 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13422 .await
13423 .is_err()
13424 );
13425 assert!(!started.exists(), "pre-gate must wait for the cache too");
13426 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13430 let editor = crate::repo_lock::SeatLease::checkout_with(
13431 &winner.worktree,
13432 true,
13433 crate::config::test_coordination().as_ref(),
13434 &wait,
13435 )
13436 .await
13437 .unwrap();
13438 drop(editor);
13439 guard.release();
13440 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13441 .await
13442 .unwrap()
13443 .unwrap();
13444 drop(gate);
13445 assert!(outcomes.iter().all(CommandOutcome::ok));
13446 assert!(started.exists());
13447 }
13448
13449 #[tokio::test]
13450 #[cfg(unix)]
13451 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13452 use std::os::fd::AsRawFd as _;
13453
13454 ask_test_home();
13455 let tmp = tempfile::tempdir().unwrap();
13456 let repo = tmp.path().join("repo");
13457 std::fs::create_dir_all(&repo).unwrap();
13458 init_repo(&repo);
13459 let common = repo.join(".git");
13460 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13461 let mut runner = runner_at(RunStatus::Prep);
13462 runner.state.repo = repo.clone();
13463 runner.state.config.coordination = crate::config::test_coordination();
13464 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13465 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13466 runner.state.config.graph.advise = false;
13467 runner.roles.implementers = vec![spec("implement")];
13468 runner.roles.judges = vec![spec("judge")];
13469 runner.prep().await.unwrap();
13470 let winner = runner.state.candidates[0].clone();
13471 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13474 let completed = tmp.path().join("seat-completed");
13475 runner.roles.implementers[0].command = vec![
13476 "sh".into(),
13477 "-c".into(),
13478 format!("touch '{}'; echo completed", completed.display()),
13479 ];
13480 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13481 assert_eq!(
13482 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13483 0
13484 );
13485 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13486 let mut implement = Box::pin(runner.implement());
13487 tokio::select! {
13488 result = &mut implement => panic!("rescue must wait: {result:?}"),
13489 result = tokio::time::timeout(Duration::from_secs(10), async {
13490 while !completed.exists() {
13491 tokio::time::sleep(Duration::from_millis(20)).await;
13492 }
13493 }) => result.unwrap(),
13494 }
13495 assert!(
13496 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13497 .await
13498 .is_err()
13499 );
13500 assert_eq!(
13501 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13502 before
13503 );
13504 assert_eq!(
13505 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13506 0
13507 );
13508 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13509 .await
13510 .unwrap()
13511 .unwrap();
13512 drop(implement);
13513 assert_ne!(
13514 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13515 before
13516 );
13517 assert!(git::is_clean(&winner.worktree).await.unwrap());
13518
13519 let started = tmp.path().join("pre-gate-started");
13520 let proceed = tmp.path().join("pre-gate-proceed");
13521 let checked = tmp.path().join("gate-ran");
13522 runner.state.config.verify.pre_gate = vec![format!(
13523 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13524 started.display(),
13525 proceed.display()
13526 )];
13527 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13528 assert_eq!(
13529 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13530 0
13531 );
13532 let mut gate = Box::pin(runner.run_gate(&winner));
13533 assert!(
13534 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13535 .await
13536 .is_err()
13537 );
13538 assert!(!started.exists());
13539 assert!(!checked.exists());
13540 assert!(git::is_clean(&winner.worktree).await.unwrap());
13541 assert_eq!(
13542 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13543 0
13544 );
13545 tokio::select! {
13546 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13547 result = tokio::time::timeout(Duration::from_secs(10), async {
13548 while !started.exists() {
13549 tokio::time::sleep(Duration::from_millis(20)).await;
13550 }
13551 }) => result.unwrap(),
13552 }
13553 assert_eq!(
13554 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13555 -1,
13556 "mechanical edits must hold the real lock"
13557 );
13558 std::fs::write(proceed, "ready").unwrap();
13559 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13560 .await
13561 .unwrap()
13562 .unwrap();
13563 drop(gate);
13564 assert!(outcomes.iter().all(CommandOutcome::ok));
13565 assert!(checked.exists());
13566 assert!(runner.state.pre_gate_commit.is_some());
13567 assert!(git::is_clean(&winner.worktree).await.unwrap());
13568 assert_eq!(
13569 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13570 0,
13571 "gate completion must release coordination"
13572 );
13573 drop(outsider);
13574 std::fs::remove_file(common.join("updater.lock")).unwrap();
13575 std::fs::create_dir(common.join("updater.lock")).unwrap();
13576 std::fs::remove_file(&started).unwrap();
13577 std::fs::remove_file(&checked).unwrap();
13578 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13579 let error = runner.run_gate(&winner).await.unwrap_err();
13580 assert!(
13581 error
13582 .to_string()
13583 .contains("acquire mandatory repository lock")
13584 );
13585 assert!(
13586 !started.exists(),
13587 "failed acquisition must prevent pre-gate"
13588 );
13589 assert!(!checked.exists(), "failed acquisition must prevent gate");
13590 assert!(git::is_clean(&winner.worktree).await.unwrap());
13591 assert_eq!(
13592 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13593 head
13594 );
13595 }
13596
13597 #[tokio::test]
13598 #[cfg(unix)]
13599 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13600 use std::os::fd::AsRawFd as _;
13601 use std::time::Duration;
13602
13603 ask_test_home();
13604 let tmp = tempfile::tempdir().unwrap();
13605 let repo = tmp.path().join("repo");
13606 std::fs::create_dir_all(&repo).unwrap();
13607 init_repo(&repo);
13608 let common = repo.join(".git");
13609 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13610 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13611 assert_eq!(
13612 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13613 0
13614 );
13615 let mut runner = runner_at(RunStatus::Prep);
13616 runner.state.repo = repo.clone();
13617 runner.state.config.coordination = crate::config::test_coordination();
13618 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13619 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13620 runner.state.config.graph.advise = true;
13621 runner.state.config.graph.advisors = 1;
13622 runner.roles.implementers = vec![spec("implement")];
13623 runner.roles.judges = vec![spec("judge")];
13624 let root = runner.state.worktree_root();
13625 let started = tmp.path().join("setup-started");
13626 let proceed = tmp.path().join("setup-proceed");
13627 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13628 run: Some(format!(
13629 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13630 started.display(),
13631 proceed.display()
13632 )),
13633 ..Default::default()
13634 }];
13635 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13636 let config = git::git(&repo, &["config", "--local", "--list"])
13637 .await
13638 .unwrap();
13639 let mut prep = Box::pin(runner.prep());
13640 assert!(
13641 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13642 .await
13643 .is_err()
13644 );
13645 assert!(
13646 !root.exists(),
13647 "contention must prevent even checkout creation"
13648 );
13649 assert!(!started.exists(), "contention must prevent setup");
13650 assert_eq!(
13651 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13652 branches
13653 );
13654 assert_eq!(
13655 git::git(&repo, &["config", "--local", "--list"])
13656 .await
13657 .unwrap(),
13658 config
13659 );
13660 drop(lock);
13661 tokio::select! {
13662 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13663 result = tokio::time::timeout(Duration::from_secs(10), async {
13664 while !started.exists() {
13665 tokio::time::sleep(Duration::from_millis(20)).await;
13666 }
13667 }) => result.unwrap(),
13668 }
13669 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13670 assert_eq!(
13671 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13672 -1
13673 );
13674 assert!(
13675 tokio::time::timeout(
13676 Duration::from_millis(100),
13677 crate::repo_lock::SeatLease::for_checkout(
13678 &repo,
13679 true,
13680 crate::config::test_coordination().as_ref()
13681 )
13682 )
13683 .await
13684 .is_err(),
13685 "preparation must also exclude local editors"
13686 );
13687 std::fs::write(proceed, "ready").unwrap();
13688 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13689 .await
13690 .unwrap()
13691 .unwrap();
13692 drop(prep);
13693 assert_eq!(runner.state.prep_complete, Some(true));
13694 assert!(root.join("judge-1").exists());
13695 assert!(root.join("advisor-1").exists());
13696 assert_eq!(
13697 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13698 0
13699 );
13700 }
13701
13702 #[tokio::test]
13703 #[cfg(unix)]
13704 async fn preparation_lock_failure_prevents_repository_changes() {
13705 ask_test_home();
13706 let tmp = tempfile::tempdir().unwrap();
13707 let repo = tmp.path().join("repo");
13708 std::fs::create_dir_all(&repo).unwrap();
13709 init_repo(&repo);
13710 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13711 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13714 let mut runner = runner_at(RunStatus::Prep);
13715 runner.state.repo = repo.clone();
13716 runner.state.config.coordination = crate::config::test_coordination();
13717 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13718 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13719 runner.roles.implementers = vec![spec("implement")];
13720 let error = runner.prep().await.unwrap_err();
13721 assert!(
13722 error
13723 .to_string()
13724 .contains("acquire mandatory repository lock")
13725 );
13726 assert!(!runner.state.worktree_root().exists());
13727 assert!(runner.state.candidates.is_empty());
13728 assert!(!repo.join(".git/config.worktree").exists());
13729 assert_eq!(
13730 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13731 "* main"
13732 );
13733 }
13734
13735 #[tokio::test]
13736 async fn preparation_records_creation_intent_before_git_can_fail() {
13737 ask_test_home();
13738 let tmp = tempfile::tempdir().unwrap();
13739 let repo = tmp.path().join("repo");
13740 std::fs::create_dir_all(&repo).unwrap();
13741 init_repo(&repo);
13742 let mut runner = runner_at(RunStatus::Prep);
13743 runner.state.repo = repo.clone();
13744 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13745 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13746 runner.state.config.blind.commit_msg_hook = false;
13747 runner.roles.implementers = vec![spec("one")];
13748 let label = blind::assign_labels(1, runner.state.seed)[0];
13749 let branch = runner.state.branch_for(label);
13750 git::git(&repo, &["branch", &branch]).await.unwrap();
13753 assert!(runner.prep().await.is_err());
13754 let saved = RunState::load(&runner.state.id).unwrap();
13755 assert!(saved.candidates.is_empty());
13756 let event = saved
13757 .events
13758 .iter()
13759 .find(|e| e.node == "prep-intent")
13760 .unwrap();
13761 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13762 assert_eq!(intent["branch"], branch);
13763 assert_eq!(intent["start"], runner.state.base_commit);
13764 assert_eq!(
13765 intent["worktree"],
13766 runner
13767 .state
13768 .worktree_root()
13769 .join(format!("cand-{label}"))
13770 .to_string_lossy()
13771 .as_ref()
13772 );
13773 assert_eq!(intent["picks"], serde_json::json!([]));
13774 }
13775
13776 #[tokio::test]
13777 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13778 ask_test_home();
13779 let tmp = tempfile::tempdir().unwrap();
13780 let repo = tmp.path().join("repo");
13781 std::fs::create_dir_all(&repo).unwrap();
13782 init_repo(&repo);
13783 let mut runner = runner_at(RunStatus::Prep);
13784 runner.state.repo = repo.clone();
13785 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13786 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13787 runner.state.sources.pick_commits.push(refs::Input {
13788 token: "missing".into(),
13789 sha: "0".repeat(40),
13790 provenance: refs::Provenance::ExplicitCli,
13791 });
13792 assert!(runner.prep().await.is_err());
13793 assert!(runner.state.candidates.is_empty());
13794 assert!(!runner.state.worktree_root().exists());
13795 }
13796
13797 #[tokio::test]
13798 async fn prose_and_legacy_inference_never_import_into_candidates() {
13799 ask_test_home();
13800 let tmp = tempfile::tempdir().unwrap();
13801 let repo = tmp.path().join("repo");
13802 std::fs::create_dir_all(&repo).unwrap();
13803 init_repo(&repo);
13804 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13805 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13806 .await
13807 .unwrap();
13808 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13809 git::git(&repo, &["add", "."]).await.unwrap();
13810 git::git(&repo, &["commit", "-m", "historical implementation"])
13811 .await
13812 .unwrap();
13813 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13814 git::git(&repo, &["checkout", "main"]).await.unwrap();
13815 for instruction in [
13816 format!("Example: magi/history/A at {source}"),
13817 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13818 format!("Never import magi/history/A or {source}"),
13819 format!("Conversation answer: I mentioned {source} only as background"),
13820 ] {
13821 let mut runner = runner_at(RunStatus::Prep);
13822 runner.state.repo = repo.clone();
13823 runner.state.base_commit = base.clone();
13824 runner.state.instruction = instruction.clone();
13825 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13826 runner.state.config.blind.commit_msg_hook = false;
13827 runner.state.seeds = vec![refs::Seed {
13828 token: "magi/history/A".into(),
13829 sha: source.clone(),
13830 kind: refs::SeedKind::Unmerged,
13831 branch: true,
13832 detail: String::new(),
13833 }];
13834 runner.roles.implementers = vec![spec("one")];
13835 assert_eq!(seeded_instruction(&runner.state), instruction);
13836 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13838 legacy.as_object_mut().unwrap().remove("sources");
13839 runner.state = serde_json::from_value(legacy).unwrap();
13840 assert_eq!(runner.state.sources, refs::Sources::default());
13841 runner.state.save().unwrap();
13842 runner.state = RunState::load(&runner.state.id).unwrap();
13843 runner.prep().await.unwrap();
13844 for candidate in &runner.state.candidates {
13845 assert_eq!(
13846 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13847 base
13848 );
13849 assert!(!candidate.worktree.join("unwanted").exists());
13850 }
13851 }
13852 }
13853
13854 #[tokio::test]
13855 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13856 ask_test_home();
13857 let tmp = tempfile::tempdir().unwrap();
13858 let repo = tmp.path().join("repo");
13859 std::fs::create_dir_all(&repo).unwrap();
13860 init_repo(&repo);
13861 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13862 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13863 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13864 git::git(&repo, &["add", "."]).await.unwrap();
13865 git::git(&repo, &["commit", "-m", "required seed"])
13866 .await
13867 .unwrap();
13868 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13869 git::git(&repo, &["checkout", "main"]).await.unwrap();
13870 let mut runner = runner_at(RunStatus::Prep);
13871 runner.state.repo = repo.clone();
13872 runner.state.base_commit = base.clone();
13873 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13874 runner.state.config.graph.advise = false;
13875 runner.state.config.blind.commit_msg_hook = false;
13876 runner.state.sources =
13877 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13878 .await
13879 .unwrap();
13880 runner.roles.implementers = vec![spec("one"), spec("two")];
13881 runner.roles.judges = vec![spec("judge")];
13882 let label = blind::assign_labels(2, runner.state.seed)[0];
13883 let branch = runner.state.branch_for(label);
13884 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13885 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13886 .await
13887 .unwrap();
13888 runner.state.candidates.push(Candidate {
13889 index: 0,
13890 label,
13891 agent: "one".into(),
13892 branch,
13893 worktree,
13894 summary: String::new(),
13895 stat: String::new(),
13896 files: 0,
13897 commits: 0,
13898 empty: false,
13899 failed: None,
13900 verified_noop: None,
13901 duration_ms: 0,
13902 folded: false,
13903 });
13904 runner.state.save().unwrap();
13905 runner.state = RunState::load(&runner.state.id).unwrap();
13907 assert_eq!(runner.state.prep_complete, Some(false));
13908 runner.prep().await.unwrap();
13909 assert_eq!(runner.state.prep_complete, Some(true));
13910 assert_eq!(runner.state.candidates.len(), 2);
13911 for candidate in &runner.state.candidates {
13912 assert_eq!(
13913 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13914 git::tree_of(&repo, &seed).await.unwrap()
13915 );
13916 assert_eq!(
13917 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13918 "required commit"
13919 );
13920 }
13921 assert!(runner.state.worktree_root().join("judge-1").exists());
13922 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13923 runner.prep().await.unwrap();
13924 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13925
13926 runner.state.prep_complete = Some(false);
13927 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13928 run: Some(format!(
13929 "test -f '{}'",
13930 tmp.path().join("dependency-ready").display()
13931 )),
13932 ..Default::default()
13933 }];
13934 assert!(runner.prep().await.is_err());
13935 assert!(runner.state.prep_failure.is_some());
13936 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13937 runner.state = RunState::load(&runner.state.id).unwrap();
13938 assert!(
13939 runner.prep().await.is_err(),
13940 "correcting the cause alone must not auto-retry"
13941 );
13942 let failed_at = runner.state.events.last().unwrap().at;
13943 assert!(
13944 runner.retry_preparation(failed_at).is_err(),
13945 "old approvals cannot unlock the hold"
13946 );
13947 let mut task = crate::queue::Task::new(
13948 "retry".into(),
13949 "retry setup".into(),
13950 repo.clone(),
13951 crate::queue::Source::Human,
13952 );
13953 task.start(runner.state.id.clone());
13954 task.hold_machine(Some("setup failed".into()));
13955 task.release_operator("cli", None);
13956 runner
13957 .retry_preparation(task.resumption.as_ref().unwrap().at)
13958 .unwrap();
13959 runner.prep().await.unwrap();
13960 assert!(runner.state.prep_failure.is_none());
13961 assert_eq!(runner.state.prep_complete, Some(true));
13962 assert_eq!(runner.state.candidates.len(), 2);
13963 assert!(runner.state.worktree_root().join("judge-1").exists());
13964 for candidate in &runner.state.candidates {
13965 assert_eq!(
13966 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13967 git::tree_of(&repo, &seed).await.unwrap()
13968 );
13969 }
13970 }
13971
13972 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13973 let mut heads = Vec::new();
13974 for candidate in candidates {
13975 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13976 }
13977 heads
13978 }
13979
13980 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13981 ask_test_home();
13982 let tmp = tempfile::tempdir().unwrap();
13983 let repo = tmp.path().join("repo");
13984 std::fs::create_dir(&repo).unwrap();
13985 init_repo(&repo);
13986 let mut runner = runner_at(RunStatus::Reviewing);
13987 runner.state.repo = repo.clone();
13988 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13989 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13990 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13991 let branch = format!("candidate-{label}");
13992 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13993 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13994 .await
13995 .unwrap();
13996 if label == 'B' {
13997 std::fs::write(worktree.join("change"), "winner").unwrap();
13998 git::rescue_commit(&worktree, "candidate change")
13999 .await
14000 .unwrap();
14001 }
14002 runner.state.candidates.push(Candidate {
14003 index,
14004 label,
14005 agent: "alpha".into(),
14006 branch,
14007 worktree,
14008 summary: "completed reply".into(),
14009 stat: String::new(),
14010 files: usize::from(label == 'B'),
14011 commits: usize::from(label == 'B'),
14012 empty: label == 'A',
14013 failed: None,
14014 verified_noop: None,
14015 duration_ms: 1,
14016 folded: false,
14017 });
14018 }
14019 runner.state.judge_skipped = true;
14020 runner.state.tally = Some(Tally {
14021 first_choice: BTreeMap::new(),
14022 borda: BTreeMap::new(),
14023 winner: 'B',
14024 rankings: 0,
14025 unanimous_initial: false,
14026 deliberated: false,
14027 changed_votes: 0,
14028 unanimous_final: false,
14029 tie_break: None,
14030 judges: 0,
14031 present: 0,
14032 quorum: 0,
14033 met_quorum: true,
14034 uncontested: Some("only B produced a change".into()),
14035 });
14036 runner.state.save().unwrap();
14037 (tmp, runner)
14038 }
14039
14040 #[tokio::test]
14041 #[cfg(unix)]
14042 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14043 for (solo, failed_checkpoint) in
14044 [(true, false), (false, false), (true, true), (false, true)]
14045 {
14046 let (_tmp, mut runner) = preservation_resume_fixture().await;
14047 if solo {
14048 runner.state.candidates.retain(|c| c.label == 'A');
14049 }
14050 runner.state.candidates[0].empty = false;
14051 runner.state.candidates[0].summary.clear();
14052 runner.state.tally = None;
14053 runner.state.judge_skipped = false;
14054 runner.state.status = RunStatus::Implementing;
14055 runner.state.config.disk.min_free_bytes = u64::MAX;
14056 let mut implementer = spec("alpha");
14057 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14058 runner.roles.implementers = vec![implementer];
14059 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14060 if failed_checkpoint {
14061 std::fs::create_dir_all(&patch).unwrap();
14062 }
14063 runner.state.save().unwrap();
14064 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14065 runner.state = RunState::load(&runner.state.id).unwrap();
14066 assert_eq!(runner.state.status, RunStatus::Blocked);
14067 assert!(runner.state.interrupted_implementations.contains(&'A'));
14068 assert_eq!(runner.state.candidates[0].commits, 0);
14069 assert!(!runner.state.candidates[0].empty);
14070 assert!(runner.state.candidates[0].failed.is_none());
14071 if failed_checkpoint {
14072 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14073 std::fs::remove_dir(patch).unwrap();
14074 }
14075 runner.state.resource_block = None;
14076 runner.state.config.disk.min_free_bytes = 0;
14077 runner.roles.implementers[0].command = vec![
14078 "sh".into(),
14079 "-c".into(),
14080 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14081 ];
14082 runner.implement().await.unwrap();
14083 assert!(runner.state.interrupted_implementations.is_empty());
14084 assert!(runner.state.candidate_preservation.is_empty());
14085 assert_eq!(runner.state.candidates[0].commits, 1);
14086 assert!(!runner.state.candidates[0].empty);
14087 assert!(
14088 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14089 .unwrap()
14090 .contains("retried empty interrupted seat")
14091 );
14092 if !solo {
14093 assert_eq!(runner.state.candidates[1].commits, 1);
14094 }
14095 }
14096 }
14097
14098 #[tokio::test]
14099 #[cfg(unix)]
14100 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14101 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14102 let (_tmp, mut runner) = preservation_resume_fixture().await;
14103 runner.state.candidates.retain(|c| c.label == 'A');
14104 runner.state.candidates[0].empty = false;
14105 runner.state.candidates[0].summary.clear();
14106 runner.state.tally = None;
14107 runner.state.judge_skipped = false;
14108 runner.state.status = RunStatus::Implementing;
14109 runner.state.config.disk.min_free_bytes = u64::MAX;
14112 let wt = runner.state.candidates[0].worktree.clone();
14113 std::fs::write(
14114 wt.join("interrupted-source"),
14115 "retain implementation work\n",
14116 )
14117 .unwrap();
14118 if committed {
14119 git::rescue_commit(&wt, "implementation checkpoint")
14120 .await
14121 .unwrap();
14122 }
14123 let lock = git::git(
14124 &wt,
14125 &[
14126 "rev-parse",
14127 "--path-format=absolute",
14128 "--git-path",
14129 "index.lock",
14130 ],
14131 )
14132 .await
14133 .unwrap();
14134 if block_rescue {
14135 std::fs::write(&lock, "fixture-owned lock").unwrap();
14136 }
14137 let mut implementer = spec("alpha");
14138 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14139 runner.roles.implementers = vec![implementer];
14140 runner.state.save().unwrap();
14141 let result = runner.implement().await;
14142 assert_eq!(result.is_err(), block_rescue);
14143 assert_eq!(
14144 runner.state.resource_block.as_ref().unwrap().kind,
14145 "disk pressure"
14146 );
14147 assert!(
14148 runner
14149 .state
14150 .events
14151 .iter()
14152 .any(|e| e.message.contains("agent execution interrupted"))
14153 );
14154 assert!(
14155 runner
14156 .state
14157 .dir()
14158 .join("artifacts/candidate-A-recovery/staged.patch")
14159 .exists()
14160 );
14161 if block_rescue {
14162 let saved = RunState::load(&runner.state.id).unwrap();
14163 assert!(saved.candidate_preservation.contains_key(&'A'));
14164 assert!(!saved.candidates[0].empty);
14165 assert_eq!(
14166 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14167 "retain implementation work\n"
14168 );
14169 std::fs::remove_file(lock).unwrap();
14170 runner.state = saved;
14171 } else {
14172 assert_eq!(runner.state.status, RunStatus::Blocked);
14173 assert_eq!(runner.state.candidates[0].commits, 1);
14174 }
14175 runner.roles.implementers.clear();
14178 runner.state.config.disk.min_free_bytes = 0;
14179 runner.state.resource_block = None;
14180 runner.implement().await.unwrap();
14181 assert!(runner.state.candidate_preservation.is_empty());
14182 assert_eq!(runner.state.candidates[0].commits, 1);
14183 assert!(!runner.state.candidates[0].empty);
14184 assert!(git::is_clean(&wt).await.unwrap());
14185 assert!(
14186 crate::run::read_artifact(&runner.state, "cand-A.patch")
14187 .unwrap()
14188 .contains("retain implementation work")
14189 );
14190 }
14191 }
14192
14193 #[tokio::test]
14194 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14195 let (_tmp, mut runner) = preservation_resume_fixture().await;
14196 runner.implement().await.unwrap();
14198 assert!(runner.state.candidate_preservation.is_empty());
14199 assert!(
14200 !runner
14201 .state
14202 .dir()
14203 .join("artifacts/candidate-A-recovery")
14204 .exists()
14205 );
14206 runner.fold_losers().await.unwrap();
14207 assert!(!runner.state.candidates[0].worktree.exists());
14208 for schema in [14, crate::run::SCHEMA] {
14209 runner.state.schema = schema;
14210 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14211 runner.state.save().unwrap();
14212 runner.state = RunState::load(&runner.state.id).unwrap();
14213 runner.implement().await.unwrap();
14214 assert!(runner.state.candidate_preservation.is_empty());
14215 assert!(runner.state.candidates[0].folded);
14216 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14217 }
14218 }
14219
14220 #[tokio::test]
14221 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14222 let (_tmp, mut runner) = preservation_resume_fixture().await;
14223 let wt = runner.state.candidates[0].worktree.clone();
14224 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14225 runner.state.schema = 14;
14226 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14227 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14228 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14229 for _ in 0..2 {
14230 assert!(runner.implement().await.is_err());
14231 assert_eq!(runner.state.status, RunStatus::Blocked);
14232 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14233 assert!(
14234 runner.state.candidates[0].empty,
14235 "old verdict must remain unusable"
14236 );
14237 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14238 assert!(runner.state.judge_skipped);
14239 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14240 assert!(
14241 crate::run::read_artifact(&runner.state, "cand-A.patch")
14242 .unwrap()
14243 .contains("owned edits")
14244 );
14245 assert!(wt.exists());
14246 assert_eq!(
14247 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14248 .await
14249 .unwrap(),
14250 1
14251 );
14252 runner.state = RunState::load(&runner.state.id).unwrap();
14253 }
14254 }
14255
14256 #[tokio::test]
14257 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14258 let (_tmp, mut runner) = preservation_resume_fixture().await;
14259 let wt = runner.state.candidates[0].worktree.clone();
14260 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14261 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14262 let index_lock = PathBuf::from(
14263 git::git(
14264 &wt,
14265 &[
14266 "rev-parse",
14267 "--path-format=absolute",
14268 "--git-path",
14269 "index.lock",
14270 ],
14271 )
14272 .await
14273 .unwrap(),
14274 );
14275 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14276 runner.state.schema = 14;
14277 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14278 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14279 assert!(runner.implement().await.is_err());
14280 assert!(runner.after_implement().is_err());
14281 assert!(runner.fold_losers().await.is_err());
14282 assert_eq!(
14283 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14284 "failed seat edits"
14285 );
14286 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14287 runner.state = RunState::load(&runner.state.id).unwrap();
14288 std::fs::remove_file(index_lock).unwrap();
14289 assert!(runner.implement().await.is_err());
14291 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14292 assert!(runner.state.candidates[0].empty);
14293 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14294 assert!(
14295 crate::run::read_artifact(&runner.state, "cand-A.patch")
14296 .unwrap()
14297 .contains("failed seat edits")
14298 );
14299 assert!(wt.exists());
14300 }
14301
14302 #[tokio::test]
14303 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14304 let (_tmp, mut runner) = preservation_resume_fixture().await;
14305 let repo = runner.state.repo.clone();
14306 let common = git::git(
14307 &repo,
14308 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14309 )
14310 .await
14311 .unwrap();
14312 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14313 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14314 assert!(runner.fold_losers().await.is_err());
14315 assert!(!runner.state.candidates[0].worktree.exists());
14316 assert!(!runner.state.candidates[0].folded);
14317 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14318 runner.state = RunState::load(&runner.state.id).unwrap();
14319 std::fs::remove_file(branch_lock).unwrap();
14320 let judge = runner.state.worktree_root().join("judge-1");
14321 git::worktree_add_detached(&repo, &judge, "main")
14322 .await
14323 .unwrap();
14324 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14325 runner.roles.judges = vec![spec("judge")];
14326 assert!(runner.fold_losers().await.is_err());
14327 runner.state = RunState::load(&runner.state.id).unwrap();
14328 assert!(
14329 runner.state.candidates[0].folded,
14330 "candidate progress saved before judge failure"
14331 );
14332 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14333 assert!(judge.join("uncommitted").exists());
14334 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14335 runner.fold_losers().await.unwrap();
14336 assert!(!judge.exists());
14337 runner.fold_losers().await.unwrap();
14338 }
14339
14340 #[tokio::test]
14341 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14342 let (_tmp, mut runner) = preservation_resume_fixture().await;
14343 runner.state.candidates.retain(|c| c.label == 'B');
14344 runner.state.tally = None;
14345 runner.state.judge_skipped = false;
14346 runner.state.prep_complete = Some(true);
14347 runner.state.config.graph.advise = false;
14348 runner.state.config.graph.selected_implementer = Some("alpha".into());
14349 let wt = runner.state.candidates[0].worktree.clone();
14350 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14351 runner
14352 .state
14353 .candidate_preservation
14354 .insert('B', "awaiting persistence".into());
14355 let index_lock = git::git(
14356 &wt,
14357 &[
14358 "rev-parse",
14359 "--path-format=absolute",
14360 "--git-path",
14361 "index.lock",
14362 ],
14363 )
14364 .await
14365 .unwrap();
14366 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14367 runner.state.save().unwrap();
14368 assert!(runner.execute_graph().await.is_err());
14369 assert_eq!(runner.state.status, RunStatus::Blocked);
14370 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14371 std::fs::remove_file(index_lock).unwrap();
14372 for _ in 0..2 {
14375 runner.state = RunState::load(&runner.state.id).unwrap();
14376 runner.pause.park();
14377 runner.execute_graph().await.unwrap();
14378 assert_eq!(runner.state.status, RunStatus::Implementing);
14379 assert!(runner.state.candidate_preservation.is_empty());
14380 assert_eq!(runner.state.candidates[0].commits, 2);
14381 assert!(runner.state.tally.is_none());
14382 }
14383 runner.implement().await.unwrap();
14384 assert_eq!(runner.state.status, RunStatus::Judging);
14385 runner.state.status = RunStatus::Blocked;
14387 runner.state.candidates[0].failed = Some("implementation failed".into());
14388 runner.state.save().unwrap();
14389 let error = runner.execute_graph().await.unwrap_err();
14390 assert!(
14391 error
14392 .to_string()
14393 .contains("selected implementation is held")
14394 );
14395 }
14396
14397 #[tokio::test]
14398 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14399 let (_tmp, mut runner) = preservation_resume_fixture().await;
14400 let repo = runner.state.repo.clone();
14401 let candidate = runner.state.candidates[0].clone();
14402 let registered = git::worktree_holding(&repo, &candidate.branch)
14405 .await
14406 .unwrap()
14407 .unwrap();
14408 assert_eq!(
14409 registered.canonicalize().unwrap(),
14410 candidate.worktree.canonicalize().unwrap()
14411 );
14412 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14413 assert_eq!(
14414 git::worktree_holding(&repo, &candidate.branch)
14415 .await
14416 .unwrap(),
14417 Some(registered)
14418 );
14419 assert!(
14420 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14421 "the stale registration initially keeps the branch checked out"
14422 );
14423 runner.fold_losers().await.unwrap();
14424 assert!(runner.state.candidates[0].folded);
14425 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14426 assert_eq!(
14427 git::worktree_holding(&repo, &candidate.branch)
14428 .await
14429 .unwrap(),
14430 None
14431 );
14432 runner.state = RunState::load(&runner.state.id).unwrap();
14433 runner.fold_losers().await.unwrap();
14434 }
14435
14436 #[tokio::test]
14437 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14438 let (_tmp, mut runner) = preservation_resume_fixture().await;
14439 let repo = runner.state.repo.clone();
14440 let candidate = runner.state.candidates[0].clone();
14441 let winner = runner.state.candidates[1].clone();
14442 let registered = git::worktree_holding(&repo, &candidate.branch)
14443 .await
14444 .unwrap()
14445 .unwrap();
14446 assert_eq!(
14447 registered.canonicalize().unwrap(),
14448 candidate.worktree.canonicalize().unwrap()
14449 );
14450 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14451 assert_eq!(
14452 git::worktree_holding(&repo, &candidate.branch)
14453 .await
14454 .unwrap(),
14455 Some(registered)
14456 );
14457 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14458
14459 for _ in 0..2 {
14460 fold_run(&mut runner.state, false, &crate::run::home())
14461 .await
14462 .unwrap();
14463 runner.state = RunState::load(&runner.state.id).unwrap();
14464 assert!(runner.state.candidates[0].folded);
14465 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14466 assert_eq!(
14467 git::worktree_holding(&repo, &candidate.branch)
14468 .await
14469 .unwrap(),
14470 None
14471 );
14472 assert!(!runner.state.candidates[1].folded);
14473 assert!(winner.worktree.exists());
14474 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14475 }
14476 }
14477
14478 #[tokio::test]
14479 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14480 ask_test_home();
14481 let tmp = tempfile::tempdir().unwrap();
14482 let repo = tmp.path().join("repo");
14483 std::fs::create_dir(&repo).unwrap();
14484 init_repo(&repo);
14485 let mut runner = runner_at(RunStatus::Implementing);
14486 runner.state.repo = repo.clone();
14487 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14488 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14489 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14490 let branch = format!("candidate-{label}");
14491 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14492 git::worktree_add_branch(&repo, &wt, &branch, "main")
14493 .await
14494 .unwrap();
14495 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14496 git::git(&wt, &["add", "README.md"]).await.unwrap();
14497 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14498 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14499 runner.state.candidates.push(Candidate {
14500 index,
14501 label,
14502 agent: "alpha".to_owned(),
14503 branch,
14504 worktree: wt,
14505 summary: "completed implementation".to_owned(),
14506 stat: String::new(),
14507 files: 0,
14508 commits: 0,
14509 empty: false,
14510 failed: None,
14511 verified_noop: None,
14512 duration_ms: 42,
14513 folded: false,
14514 });
14515 runner
14516 .state
14517 .candidate_preservation
14518 .insert(label, "awaiting persistence".to_owned());
14519 }
14520 let wt = runner.state.candidates[0].worktree.clone();
14521 let index_lock = git::git(
14522 &wt,
14523 &[
14524 "rev-parse",
14525 "--path-format=absolute",
14526 "--git-path",
14527 "index.lock",
14528 ],
14529 )
14530 .await
14531 .unwrap();
14532 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14533 runner.state.save().unwrap();
14534 assert!(runner.persist_candidates().await.is_err());
14535 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14536 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14537 assert_eq!(
14538 runner.state.candidates[1].commits, 1,
14539 "other results are still collected"
14540 );
14541 assert!(
14542 !runner.state.candidates[0].empty,
14543 "failed rescue is not no change"
14544 );
14545 assert_eq!(
14546 runner.state.candidates[0].summary,
14547 "completed implementation"
14548 );
14549 assert!(runner.after_implement().is_err());
14550 let home = tmp.path().join("fold-home");
14551 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14552 assert!(wt.join("new-file").exists());
14553 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14554 assert!(recovery.join("staged.patch").exists());
14555 assert!(recovery.join("unstaged.patch").exists());
14556 assert_eq!(
14557 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14558 "untracked edits\n"
14559 );
14560 runner.state.begin_pass();
14561 runner.state.end_pass_parked(ParkCause::Upgrade {
14562 from: "1.0".into(),
14563 to: Some("1.1".into()),
14564 });
14565 runner.state.save().unwrap();
14566 runner.state = RunState::load(&runner.state.id).unwrap();
14567 assert!(matches!(
14568 runner.state.passes.last().unwrap().end,
14569 Some(crate::run::PassEnd::Parked {
14570 cause: ParkCause::Upgrade { .. }
14571 })
14572 ));
14573 std::fs::remove_file(index_lock).unwrap();
14574 runner.implement().await.unwrap();
14576 assert!(runner.state.candidate_preservation.is_empty());
14577 assert!(
14578 runner
14579 .state
14580 .candidates
14581 .iter()
14582 .all(|c| c.commits == 1 && !c.empty)
14583 );
14584 assert!(git::is_clean(&wt).await.unwrap());
14585 assert!(
14586 crate::run::read_artifact(&runner.state, "cand-A.patch")
14587 .unwrap()
14588 .contains("later tracked edits")
14589 );
14590 }
14591
14592 #[tokio::test]
14593 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14594 ask_test_home();
14595 let tmp = tempfile::tempdir().unwrap();
14596 let repo = tmp.path().join("repo");
14597 std::fs::create_dir(&repo).unwrap();
14598 init_repo(&repo);
14599 let wt = tmp.path().join("candidate");
14600 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14601 .await
14602 .unwrap();
14603 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14604 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14605 assert!(wt.join("uncommitted").exists());
14606 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14607 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14608 let mut runner = runner_at(RunStatus::Implementing);
14609 runner.state.repo = repo.clone();
14610 runner.state.base_commit = "missing-base".to_owned();
14611 runner.state.candidates.push(Candidate {
14612 index: 0,
14613 label: 'A',
14614 agent: "alpha".into(),
14615 branch: "candidate".into(),
14616 worktree: wt.clone(),
14617 summary: "completed".into(),
14618 stat: String::new(),
14619 files: 0,
14620 commits: 0,
14621 empty: false,
14622 failed: None,
14623 verified_noop: None,
14624 duration_ms: 1,
14625 folded: false,
14626 });
14627 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14628 assert!(
14629 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14630 .await
14631 .is_err()
14632 );
14633 assert!(!runner.state.candidates[0].folded);
14634 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14635 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14636 runner
14637 .state
14638 .candidate_preservation
14639 .insert('A', "pending".into());
14640 assert!(runner.persist_candidates().await.is_err());
14641 assert!(!runner.state.candidates[0].empty);
14642 assert!(!runner.state.candidates[0].folded);
14643 assert!(wt.exists());
14644 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14645 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14646 std::fs::create_dir_all(&patch_path).unwrap();
14647 assert!(
14648 runner.persist_candidates().await.is_err(),
14649 "artifact failure stays pending"
14650 );
14651 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14652 assert!(!runner.state.candidates[0].empty);
14653 std::fs::remove_dir(patch_path).unwrap();
14654 runner.persist_candidates().await.unwrap();
14655 assert!(
14656 runner.state.candidates[0].empty,
14657 "only successful inspection confirms no change"
14658 );
14659 }
14660
14661 fn init_repo(dir: &Path) {
14662 let run = |args: &[&str]| {
14663 let out = std::process::Command::new("git")
14664 .args(args)
14665 .current_dir(dir)
14666 .quiet()
14667 .output()
14668 .expect("spawn git");
14669 assert!(
14670 out.status.success(),
14671 "git {args:?} failed: {}",
14672 String::from_utf8_lossy(&out.stderr)
14673 );
14674 };
14675 run(&["init", "-b", "main"]);
14676 run(&["config", "user.name", "magi test"]);
14677 run(&["config", "user.email", "magi@example.com"]);
14678 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14679 run(&["add", "-A"]);
14680 run(&["commit", "-m", "init"]);
14681 }
14682
14683 fn ask_test_home() {
14691 crate::run::pin_test_home();
14692 }
14693
14694 fn runner_at(status: RunStatus) -> Runner {
14697 let mut state = RunState::new(
14698 PathBuf::from("/nonexistent/repo"),
14699 "main".to_owned(),
14700 "deadbeef".to_owned(),
14701 "task".to_owned(),
14702 Config::default(),
14703 );
14704 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14707 state.id = format!(
14708 "20000101-000000-{:04x}",
14709 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14710 );
14711 state.status = status;
14712 Runner {
14713 state,
14714 roles: ResolvedRoles {
14715 implementers: Vec::new(),
14716 judges: Vec::new(),
14717 reviewers: Vec::new(),
14718 fixer: None,
14719 conductor: conductor(),
14720 implementer_roster: Vec::new(),
14721 judge_roster: Vec::new(),
14722 reviewer_roster: Vec::new(),
14723 },
14724 sem: Arc::new(Semaphore::new(1)),
14725 pause: Pause::new(),
14726 interrupt: Pause::new(),
14727 }
14728 }
14729
14730 #[tokio::test]
14731 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14732 crate::run::pin_test_home();
14733 let mut runner = runner_at(RunStatus::Gating);
14734 runner.state.save().unwrap();
14735 let id = runner.state.id.clone();
14736 let passes = runner.state.passes.len();
14737 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14738 let mut claiming = Box::pin(runner.claim_driver());
14739 std::future::poll_fn(|cx| {
14740 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14741 std::task::Poll::Ready(())
14742 })
14743 .await;
14744 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14745 drop(owner);
14746 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14747 .await
14748 .unwrap()
14749 .unwrap();
14750 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14751 assert!(execution.runner.state.resource_block.is_none());
14752 assert_eq!(
14753 execution.runner.state.liveness(false),
14754 crate::run::Liveness::Live
14755 );
14756 drop(execution);
14757 assert!(RunState::load(&id).unwrap().driver_exited);
14758 }
14759
14760 #[tokio::test]
14761 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14762 crate::run::pin_test_home();
14763 let mut runner = runner_at(RunStatus::Gating);
14764 runner.state.save().unwrap();
14765 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14768 assert!(runner.claim_driver().await.is_err());
14769 assert!(runner.state.driver_exited);
14770 assert_eq!(runner.state.driver_pid, None);
14771 let disk = RunState::load(&runner.state.id).unwrap();
14772 assert!(disk.driver_exited);
14773 assert_eq!(disk.driver_pid, None);
14774 assert_eq!(disk.status, RunStatus::Gating);
14775 }
14776
14777 #[tokio::test]
14778 async fn an_early_graph_error_releases_its_driver() {
14779 crate::run::pin_test_home();
14780 let mut runner = runner_at(RunStatus::Blocked);
14781 runner.state.config.graph.selected_implementer = Some("selected".into());
14782 runner.state.save().unwrap();
14783 assert!(runner.execute().await.is_err());
14784 let disk = RunState::load(&runner.state.id).unwrap();
14785 assert!(disk.driver_exited);
14786 assert_eq!(disk.driver_pid, None);
14787 assert_eq!(disk.status, RunStatus::Blocked);
14788 }
14789
14790 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14791 crate::run::MergeRepair {
14792 question: "repair-question".into(),
14793 head: "old-head".into(),
14794 pr: "https://github.com/o/r/pull/42".into(),
14795 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14796 gate: vec![outcome(Some(0), false)],
14797 gate_head: Some("old-head".into()),
14798 gate_fixes: Vec::new(),
14799 pre_gate: Vec::new(),
14800 phase,
14801 failure: None,
14802 }
14803 }
14804
14805 #[tokio::test]
14806 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14807 crate::run::pin_test_home();
14808 let mut runner = runner_at(RunStatus::Landing);
14809 runner.state.merge_repairs.push(merge_repair_request(
14810 crate::run::MergeRepairPhase::Requested,
14811 ));
14812 runner.state.save().unwrap();
14813 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14814 assert!(
14815 runner
14816 .execute()
14817 .await
14818 .unwrap_err()
14819 .to_string()
14820 .contains("already running")
14821 );
14822 assert_eq!(
14823 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14824 crate::run::MergeRepairPhase::Requested
14825 );
14826 drop(claim);
14827 let mut disk = runner.state.clone();
14828 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14829 disk.save().unwrap();
14830 assert!(
14831 runner
14832 .execute()
14833 .await
14834 .unwrap_err()
14835 .to_string()
14836 .contains("decision changed")
14837 );
14838 assert_eq!(
14839 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14840 crate::run::MergeRepairPhase::Complete
14841 );
14842 }
14843
14844 #[test]
14845 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14846 crate::run::pin_test_home();
14847 let mut runner = runner_at(RunStatus::Landing);
14848 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14849 runner.state.reviews = request.reviews.clone();
14850 runner.state.gate = request.gate.clone();
14851 runner.state.gate_ran = true;
14852 runner.state.land_approval = Some(crate::run::LandApproval {
14853 question: "old".into(),
14854 head: "old-head".into(),
14855 });
14856 runner.state.merge_repairs.push(request);
14857 runner.begin_merge_repair_review(0).unwrap();
14858 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14859 assert!(!runner.state.gate_status().ok());
14860 assert!(runner.state.land_approval.is_none());
14861 let disk = RunState::load(&runner.state.id).unwrap();
14862 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14863 assert!(disk.merge_repairs[0].gate[0].ok());
14864 assert_eq!(
14865 disk.merge_repairs[0].phase,
14866 crate::run::MergeRepairPhase::Reviewing
14867 );
14868 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14869 assert!(disk.reviews.is_empty());
14870 }
14871
14872 #[tokio::test]
14873 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14874 crate::run::pin_test_home();
14875 for phase in [
14876 crate::run::MergeRepairPhase::Fixing,
14877 crate::run::MergeRepairPhase::Requested,
14878 ] {
14879 let mut runner = runner_at(RunStatus::Landing);
14880 runner.state.config.graph.land_rounds = 0;
14881 runner.state.merge_repairs.push(merge_repair_request(phase));
14882 runner.state.save().unwrap();
14883 assert!(!runner.repair_merge_review().await.unwrap());
14884 assert_eq!(runner.state.status, RunStatus::Blocked);
14885 assert_eq!(
14886 runner.state.merge_repairs[0].phase,
14887 crate::run::MergeRepairPhase::Failed
14888 );
14889 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14890 assert!(
14891 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14892 "interrupted"
14893 } else {
14894 "budget exhausted"
14895 })
14896 );
14897 assert!(runner.state.merge.is_none());
14898 }
14899 }
14900
14901 #[test]
14905 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14906 crate::run::pin_test_home();
14907 let mut runner = runner_at(RunStatus::Implementing);
14908 let interrupt = Pause::new();
14909 runner.watch_interrupt(interrupt.clone());
14910
14911 interrupt.park_because("task a1b2 asked to run first");
14912
14913 assert!(runner.park_here().expect("park_here"));
14914 assert!(runner.state.parked);
14915 let last = runner.state.events.last().expect("a park event");
14916 assert_eq!(last.node, "park");
14917 assert!(
14918 last.message.contains("task a1b2 asked to run first"),
14919 "expected the interrupt reason in {:?}",
14920 last.message
14921 );
14922 }
14923
14924 #[test]
14927 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14928 crate::run::pin_test_home();
14929 let mut runner = runner_at(RunStatus::Implementing);
14930 runner.state.begin_pass();
14931 let shutdown = Pause::new();
14932 runner.on_pause(shutdown.clone());
14933 let upgrade = ParkCause::Upgrade {
14934 from: "0.121.0".to_owned(),
14935 to: Some("0.121.1".to_owned()),
14936 };
14937 shutdown.park_for(upgrade.clone());
14938 shutdown.park_for(ParkCause::Operator);
14939
14940 assert!(runner.park_here().expect("park_here"));
14941 assert_eq!(
14942 runner.state.passes[0].end,
14943 Some(crate::run::PassEnd::Parked { cause: upgrade })
14944 );
14945 let last = runner.state.events.last().expect("a park event");
14946 assert!(last.message.contains("0.121.0"), "{}", last.message);
14947 }
14948
14949 #[test]
14957 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14958 crate::run::pin_test_home();
14959 let mut runner = runner_at(RunStatus::Implementing);
14960 let shutdown = Pause::new();
14961 runner.on_pause(shutdown.clone());
14962 let interrupt = Pause::new();
14963 runner.watch_interrupt(interrupt.clone());
14964
14965 assert!(!runner.park_here().expect("park_here"));
14967 assert!(!runner.state.parked);
14968
14969 interrupt.park_because("test");
14971 assert!(!shutdown.parked());
14972 assert!(runner.park_here().expect("park_here"));
14973 }
14974
14975 #[tokio::test]
14989 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14990 crate::run::pin_test_home();
14991 let mut runner = runner_at(RunStatus::Implementing);
14992 let interrupt = Pause::new();
14993 runner.watch_interrupt(interrupt.clone());
14994
14995 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14996 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14997
14998 let node = async move {
15002 started_tx.send(()).expect("send started");
15003 finish_rx.await.expect("recv finish");
15004 "node finished"
15005 };
15006
15007 let interrupter = async move {
15008 started_rx.await.expect("recv started");
15009 interrupt.park_because("higher-priority task waiting");
15011 tokio::task::yield_now().await;
15015 finish_tx.send(()).expect("send finish");
15016 };
15017
15018 let (node_result, ()) = tokio::join!(node, interrupter);
15019 assert_eq!(
15020 node_result, "node finished",
15021 "the in-flight call ran to completion"
15022 );
15023
15024 assert!(runner.park_here().expect("park_here"));
15027 assert!(runner.state.parked);
15028 }
15029
15030 #[test]
15036 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15037 crate::run::pin_test_home();
15038 let mut runner = runner_at(RunStatus::Judging);
15039 runner.state.config.agents = vec![conductor()];
15043 runner.state.candidates = vec![Candidate {
15044 index: 0,
15045 label: 'A',
15046 agent: "alpha".to_owned(),
15047 branch: "magi/x/A".to_owned(),
15048 worktree: PathBuf::from("/nonexistent/worktree"),
15049 summary: "did the thing".to_owned(),
15050 stat: "1 file changed".to_owned(),
15051 files: 1,
15052 commits: 1,
15053 empty: false,
15054 failed: None,
15055 verified_noop: None,
15056 duration_ms: 1234,
15057 folded: false,
15058 }];
15059 let run_id = runner.state.id.clone();
15060
15061 let interrupt = Pause::new();
15062 runner.watch_interrupt(interrupt.clone());
15063 interrupt.park_because("task c3d4 asked to run first");
15064 assert!(runner.park_here().expect("park_here"));
15065
15066 let resumed = Runner::resume(&run_id).expect("resume");
15067 assert_eq!(resumed.state.candidates.len(), 1);
15068 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15069 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15070 assert_eq!(resumed.state.status, runner.state.status);
15071 assert!(
15072 resumed.state.parked,
15073 "still parked until `execute` actually walks the graph again"
15074 );
15075 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15076 }
15077
15078 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15080 let mut q = ask::Question::new(
15081 run.to_owned(),
15082 "implement".to_owned(),
15083 "impl-A".to_owned(),
15084 "Which storage backend should the cache use?".to_owned(),
15085 String::new(),
15086 vec!["SQLite".to_owned(), "Redis".to_owned()],
15087 );
15088 store.put(&mut q).unwrap();
15089 q
15090 }
15091
15092 #[test]
15093 fn a_failed_runs_open_question_is_abandoned() {
15094 ask_test_home();
15095 let store = ask::Questions::open();
15096 let mut runner = runner_at(RunStatus::Failed);
15097 let run = runner.state.id.clone();
15098 let q = ask_open_question(&store, &run);
15099
15100 runner.settle_questions();
15101
15102 let back = store.get(&q.id).unwrap();
15103 assert!(
15104 !back.status.open(),
15105 "the seat that asked died with the run; nobody is left to read an answer"
15106 );
15107 assert!(
15108 back.detail.contains(&run) && back.detail.contains("failed"),
15109 "the reason names what the run became, not just that it is gone: {}",
15110 back.detail
15111 );
15112 }
15113
15114 #[test]
15115 fn a_merged_runs_open_question_is_abandoned_too() {
15116 ask_test_home();
15117 let store = ask::Questions::open();
15118 for status in [RunStatus::Merged, RunStatus::Ready] {
15121 let mut runner = runner_at(status);
15122 let run = runner.state.id.clone();
15123 let q = ask_open_question(&store, &run);
15124
15125 runner.settle_questions();
15126
15127 let back = store.get(&q.id).unwrap();
15128 assert!(
15129 !back.status.open(),
15130 "{status:?} run's question must not outlive the run"
15131 );
15132 }
15133 }
15134
15135 #[test]
15136 fn a_still_resumable_runs_open_question_is_left_alone() {
15137 ask_test_home();
15138 let store = ask::Questions::open();
15139 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15145 let mut runner = runner_at(status);
15146 let run = runner.state.id.clone();
15147 let q = ask_open_question(&store, &run);
15148
15149 runner.settle_questions();
15150
15151 let back = store.get(&q.id).unwrap();
15152 assert!(
15153 back.status.open(),
15154 "{status:?} is still alive; the question must still be waiting"
15155 );
15156 }
15157 }
15158
15159 #[test]
15160 fn settle_questions_never_touches_an_already_answered_question() {
15161 ask_test_home();
15162 let store = ask::Questions::open();
15163 let mut runner = runner_at(RunStatus::Failed);
15164 let run = runner.state.id.clone();
15165 let mut q = ask_open_question(&store, &run);
15166 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15167 .unwrap();
15168 store.put(&mut q).unwrap();
15169
15170 runner.settle_questions();
15175 runner.settle_questions();
15176
15177 let back = store.get(&q.id).unwrap();
15178 assert_eq!(
15179 back.status,
15180 ask::QuestionStatus::Answered,
15181 "a real answer is a decision on record, never overwritten by a sweep"
15182 );
15183 }
15184
15185 #[tokio::test]
15196 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15197 crate::run::pin_test_home();
15198 let tmp = tempfile::tempdir().expect("tempdir");
15199 let repo = tmp.path().join("repo");
15200 std::fs::create_dir_all(&repo).unwrap();
15201 init_repo(&repo);
15202
15203 let mut config = Config::default();
15204 config.graph.worktree_root = Some(tmp.path().join("wt"));
15205
15206 let mut state = RunState::new(
15207 repo.clone(),
15208 "main".to_owned(),
15209 "deadbeef".to_owned(),
15210 "task".to_owned(),
15211 config,
15212 );
15213 let root = state.worktree_root();
15214 let wt_a = root.join("cand-A");
15215 let wt_b = root.join("cand-B");
15216 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15217 .await
15218 .expect("worktree A");
15219 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15220 .await
15221 .expect("worktree B");
15222
15223 state.candidates = vec![
15224 Candidate {
15225 index: 0,
15226 label: 'A',
15227 agent: "alpha".to_owned(),
15228 branch: "magi/x/A".to_owned(),
15229 worktree: wt_a.clone(),
15230 summary: String::new(),
15231 stat: String::new(),
15232 files: 0,
15233 commits: 0,
15234 empty: false,
15235 failed: None,
15236 verified_noop: None,
15237 duration_ms: 0,
15238 folded: false,
15239 },
15240 Candidate {
15241 index: 1,
15242 label: 'B',
15243 agent: "beta".to_owned(),
15244 branch: "magi/x/B".to_owned(),
15245 worktree: wt_b.clone(),
15246 summary: String::new(),
15247 stat: String::new(),
15248 files: 0,
15249 commits: 0,
15250 empty: false,
15251 failed: None,
15252 verified_noop: None,
15253 duration_ms: 0,
15254 folded: false,
15255 },
15256 ];
15257 state.tally = Some(Tally {
15258 first_choice: BTreeMap::from([('A', 1)]),
15259 borda: BTreeMap::new(),
15260 winner: 'A',
15261 rankings: 1,
15262 unanimous_initial: true,
15263 deliberated: false,
15264 changed_votes: 0,
15265 unanimous_final: true,
15266 tie_break: None,
15267 judges: 1,
15268 present: 1,
15269 quorum: 1,
15270 met_quorum: true,
15271 uncontested: None,
15272 });
15273 state.status = RunStatus::Ready;
15274
15275 fold_run(&mut state, false, &crate::run::home())
15276 .await
15277 .expect("fold_run");
15278
15279 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15280 assert!(
15281 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15282 "the unmerged winner's branch survives"
15283 );
15284 assert!(
15285 !state.candidates[0].folded,
15286 "the winner is not marked folded"
15287 );
15288
15289 assert!(!wt_b.exists(), "the loser's worktree is removed");
15290 assert!(
15291 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15292 "the loser's branch is removed"
15293 );
15294 assert!(state.candidates[1].folded, "the loser is marked folded");
15295 }
15296
15297 #[tokio::test]
15298 #[cfg(unix)]
15299 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15300 use std::os::fd::AsRawFd as _;
15301 use std::time::Duration;
15302
15303 let tmp = tempfile::tempdir().unwrap();
15304 let repo = tmp.path().join("repo");
15305 std::fs::create_dir_all(&repo).unwrap();
15306 init_repo(&repo);
15307 let common = repo.join(".git");
15308 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15309 let home = tmp.path().join("home");
15310 let mut config = Config::default();
15311 config.coordination = crate::config::test_coordination();
15312 config.graph.worktree_root = Some(tmp.path().join("wt"));
15313 config.graph.timeout_resource_wait = 1;
15314 let mut state = RunState::new(
15315 repo.clone(),
15316 "main".into(),
15317 "deadbeef".into(),
15318 "task".into(),
15319 config,
15320 );
15321 let root = state.worktree_root();
15322 let candidate = root.join("cand-A");
15323 let reviewer = root.join("review-1");
15324 let branch = state.branch_for('A');
15325 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15326 .await
15327 .unwrap();
15328 git::worktree_add_detached(&repo, &reviewer, "main")
15329 .await
15330 .unwrap();
15331 git::acquire_worktree_config(&repo).await.unwrap();
15332 state.enabled_worktree_config = true;
15333 state.status = RunStatus::Ready;
15334 state.candidates.push(Candidate {
15335 index: 0,
15336 label: 'A',
15337 agent: "implement".into(),
15338 branch: branch.clone(),
15339 worktree: candidate.clone(),
15340 summary: String::new(),
15341 stat: String::new(),
15342 files: 0,
15343 commits: 0,
15344 empty: false,
15345 failed: None,
15346 verified_noop: None,
15347 duration_ms: 0,
15348 folded: false,
15349 });
15350 state.save_under(&home).unwrap();
15351 let config_before = std::fs::read(common.join("config")).unwrap();
15352 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15353 .await
15354 .unwrap();
15355 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15358 assert_eq!(
15359 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15360 0
15361 );
15362 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15363 assert!(
15364 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15365 .await
15366 .is_err()
15367 );
15368 assert!(candidate.exists());
15369 assert!(reviewer.exists());
15370 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15371 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15372 assert_eq!(
15373 git::git(&repo, &["worktree", "list", "--porcelain"])
15374 .await
15375 .unwrap(),
15376 worktrees_before
15377 );
15378 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15379 .await
15380 .unwrap()
15381 .unwrap_err();
15382 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15383 drop(fold);
15384 assert!(candidate.exists() && reviewer.exists());
15385 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15386 assert_eq!(
15387 state.resource_block.as_ref().unwrap().kind,
15388 "repository coordination"
15389 );
15390 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15391 drop(external);
15392 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15393 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15394 .await
15395 .unwrap()
15396 .unwrap();
15397 drop(fold);
15398 assert!(removed.contains(&branch));
15399 assert!(!root.exists());
15400 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15401 assert!(state.candidates[0].folded);
15402 assert!(!state.enabled_worktree_config);
15403 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15404 let enabled = git::git_raw(
15405 &repo,
15406 &["config", "--local", "--get", "extensions.worktreeConfig"],
15407 )
15408 .await
15409 .unwrap();
15410 assert!(!enabled.ok(), "the last config reference was released");
15411 }
15412
15413 #[tokio::test]
15416 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15417 let tmp = tempfile::tempdir().expect("tempdir");
15418 let repo = tmp.path().join("repo");
15419 std::fs::create_dir_all(&repo).unwrap();
15420 init_repo(&repo);
15421 let home = tmp.path().join("home");
15422
15423 let mut config = Config::default();
15424 config.graph.worktree_root = Some(tmp.path().join("wt"));
15425 let mut state = RunState::new(
15426 repo.clone(),
15427 "main".to_owned(),
15428 "deadbeef".to_owned(),
15429 "task".to_owned(),
15430 config,
15431 );
15432 git::git(&repo, &["branch", "magi/x/A", "main"])
15434 .await
15435 .expect("branch");
15436 state.candidates = vec![Candidate {
15437 index: 0,
15438 label: 'A',
15439 agent: "alpha".to_owned(),
15440 branch: "magi/x/A".to_owned(),
15441 worktree: state.worktree_root().join("cand-A"),
15442 summary: String::new(),
15443 stat: String::new(),
15444 files: 0,
15445 commits: 0,
15446 empty: false,
15447 failed: None,
15448 verified_noop: None,
15449 duration_ms: 0,
15450 folded: true,
15451 }];
15452 state.released_to = Some("20260901-000000-new1".to_owned());
15453 state.released_branches = vec!["magi/x/A".to_owned()];
15454
15455 fold_run(&mut state, true, &home).await.expect("fold_run");
15456 state.status = RunStatus::Merged;
15457 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15458 state.merge = Some(crate::run::MergeOutcome {
15459 mode: MergeMode::Pr,
15460 ok: true,
15461 detail: "confirmed merged".into(),
15462 empty: false,
15463 });
15464 fold_run(&mut state, true, &home)
15465 .await
15466 .expect("merged fold");
15467
15468 assert!(
15469 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15470 "the handed-over branch survives a fold"
15471 );
15472 }
15473
15474 #[tokio::test]
15478 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15479 let tmp = tempfile::tempdir().expect("tempdir");
15480 let repo = tmp.path().join("repo");
15481 std::fs::create_dir_all(&repo).unwrap();
15482 init_repo(&repo);
15483 let run = |args: &[&str]| {
15484 let out = std::process::Command::new("git")
15485 .quiet()
15486 .args(args)
15487 .current_dir(&repo)
15488 .output()
15489 .expect("spawn git");
15490 assert!(out.status.success(), "git {args:?}");
15491 };
15492 run(&["branch", "magi/x/A"]);
15493 run(&["checkout", "-q", "-b", "magi/x/B"]);
15494 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15495 run(&["add", "-A"]);
15496 run(&["commit", "-q", "-m", "work"]);
15497 run(&["checkout", "-q", "main"]);
15498
15499 let probe = RunState::new(
15503 repo.clone(),
15504 "main".to_owned(),
15505 "deadbeef".to_owned(),
15506 "task".to_owned(),
15507 Config::default(),
15508 );
15509 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15510 let bare = tmp.path().join("origin.git");
15511 let out = std::process::Command::new("git")
15512 .quiet()
15513 .args(["init", "-q", "--bare"])
15514 .arg(&bare)
15515 .output()
15516 .expect("spawn git");
15517 assert!(out.status.success(), "git init --bare");
15518 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15519 run(&["push", "-q", "origin", "main"]);
15520
15521 let mut state = RunState::new(
15522 repo.clone(),
15523 "main".to_owned(),
15524 "deadbeef".to_owned(),
15525 "task".to_owned(),
15526 Config::default(),
15527 );
15528 state.seeds = vec![refs::Seed {
15529 token: "magi/27b2/A".to_owned(),
15530 kind: refs::SeedKind::Unresolved,
15531 sha: String::new(),
15532 branch: true,
15533 detail: "no branch or commit named magi/27b2/A".to_owned(),
15534 }];
15535
15536 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15537 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15538 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15539 let detail = empty_candidate_detail(&state, "main");
15540 assert!(detail.starts_with("empty candidate"), "{detail}");
15541 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15542 assert!(detail.contains("--start-from"), "{detail}");
15543 }
15544
15545 #[tokio::test]
15554 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15555 let tmp = tempfile::tempdir().expect("tempdir");
15556 let repo = tmp.path().join("repo");
15557 std::fs::create_dir_all(&repo).unwrap();
15558 init_repo(&repo);
15559
15560 let mut config = Config::default();
15561 config.merge.mode = MergeMode::Local;
15562
15563 let mut state = RunState::new(
15564 repo.clone(),
15565 "main".to_owned(),
15566 "deadbeef".to_owned(),
15567 "task".to_owned(),
15568 config,
15569 );
15570 state.candidates = vec![Candidate {
15571 index: 0,
15572 label: 'A',
15573 agent: "alpha".to_owned(),
15574 branch: "does-not-exist".to_owned(),
15575 worktree: repo.clone(),
15576 summary: String::new(),
15577 stat: String::new(),
15578 files: 0,
15579 commits: 0,
15580 empty: false,
15581 failed: None,
15582 verified_noop: None,
15583 duration_ms: 0,
15584 folded: false,
15585 }];
15586 state.tally = Some(Tally {
15587 first_choice: BTreeMap::from([('A', 1)]),
15588 borda: BTreeMap::new(),
15589 winner: 'A',
15590 rankings: 1,
15591 unanimous_initial: true,
15592 deliberated: false,
15593 changed_votes: 0,
15594 unanimous_final: true,
15595 tie_break: None,
15596 judges: 0,
15597 present: 0,
15598 quorum: 0,
15599 met_quorum: true,
15600 uncontested: Some("only candidate A produced a change".to_owned()),
15601 });
15602 state.reviews = vec![ReviewRound {
15603 round: 1,
15604 head: "deadbeef".to_owned(),
15605 verified_head: None,
15606 verified_at: None,
15607 reviews: Vec::new(),
15608 e2e: Vec::new(),
15609 fix: None,
15610 blocking: 0,
15611 answered: 0,
15612 expected: 0,
15613 clean: true,
15614 verify_retried: false,
15615 e2e_deferred: false,
15616 e2e_defer_reason: None,
15617 progressed: false,
15618 vote_split: false,
15619 reconsideration: Vec::new(),
15620 verdict: None,
15621 }];
15622 state.gate = vec![CommandOutcome {
15623 command: "test".to_owned(),
15624 code: Some(0),
15625 output_tail: String::new(),
15626 duration_ms: 0,
15627 resource_blocked: false,
15628 }];
15629 state.gate_ran = true;
15630 state.status = RunStatus::Ready;
15635 state.merge = Some(MergeOutcome {
15636 mode: MergeMode::Local,
15637 ok: false,
15638 detail: "already concluded".to_owned(),
15639 empty: false,
15640 });
15641
15642 let mut runner = Runner {
15643 state,
15644 roles: ResolvedRoles {
15645 implementers: Vec::new(),
15646 judges: Vec::new(),
15647 reviewers: Vec::new(),
15648 fixer: None,
15649 conductor: conductor(),
15650 implementer_roster: Vec::new(),
15651 judge_roster: Vec::new(),
15652 reviewer_roster: Vec::new(),
15653 },
15654 sem: Arc::new(Semaphore::new(1)),
15655 pause: Pause::new(),
15656 interrupt: Pause::new(),
15657 };
15658
15659 runner.merge().await.expect("merge");
15660
15661 assert_eq!(
15662 runner.state.status,
15663 RunStatus::Ready,
15664 "a concluded run's status must not change on reentry"
15665 );
15666 assert_eq!(
15667 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15668 Some("already concluded"),
15669 "merge must not run again once the node already recorded an outcome"
15670 );
15671 }
15672
15673 #[tokio::test]
15682 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15683 let tmp = tempfile::tempdir().expect("tempdir");
15684 let repo = tmp.path().join("repo");
15685 std::fs::create_dir_all(&repo).unwrap();
15686 init_repo(&repo);
15687
15688 let mut config = Config::default();
15689 config.merge.mode = MergeMode::Local;
15690
15691 let mut state = RunState::new(
15692 repo.clone(),
15693 "main".to_owned(),
15694 "deadbeef".to_owned(),
15695 "task".to_owned(),
15696 config,
15697 );
15698 state.candidates = vec![Candidate {
15699 index: 0,
15700 label: 'A',
15701 agent: "alpha".to_owned(),
15702 branch: "does-not-exist".to_owned(),
15703 worktree: repo.clone(),
15704 summary: String::new(),
15705 stat: String::new(),
15706 files: 0,
15707 commits: 0,
15708 empty: false,
15709 failed: None,
15710 verified_noop: None,
15711 duration_ms: 0,
15712 folded: false,
15713 }];
15714 state.tally = Some(Tally {
15715 first_choice: BTreeMap::from([('A', 1)]),
15716 borda: BTreeMap::new(),
15717 winner: 'A',
15718 rankings: 1,
15719 unanimous_initial: true,
15720 deliberated: false,
15721 changed_votes: 0,
15722 unanimous_final: true,
15723 tie_break: None,
15724 judges: 0,
15725 present: 0,
15726 quorum: 0,
15727 met_quorum: true,
15728 uncontested: Some("only candidate A produced a change".to_owned()),
15729 });
15730 state.reviews = vec![ReviewRound {
15731 round: 1,
15732 head: "deadbeef".to_owned(),
15733 verified_head: None,
15734 verified_at: None,
15735 reviews: Vec::new(),
15736 e2e: Vec::new(),
15737 fix: None,
15738 blocking: 0,
15739 answered: 0,
15740 expected: 0,
15741 clean: true,
15742 verify_retried: false,
15743 e2e_deferred: false,
15744 e2e_defer_reason: None,
15745 progressed: false,
15746 vote_split: false,
15747 reconsideration: Vec::new(),
15748 verdict: None,
15749 }];
15750 state.gate = Vec::new();
15752 state.gate_ran = false;
15753 state.status = RunStatus::Gating;
15754
15755 let mut runner = Runner {
15756 state,
15757 roles: ResolvedRoles {
15758 implementers: Vec::new(),
15759 judges: Vec::new(),
15760 reviewers: Vec::new(),
15761 fixer: None,
15762 conductor: conductor(),
15763 implementer_roster: Vec::new(),
15764 judge_roster: Vec::new(),
15765 reviewer_roster: Vec::new(),
15766 },
15767 sem: Arc::new(Semaphore::new(1)),
15768 pause: Pause::new(),
15769 interrupt: Pause::new(),
15770 };
15771
15772 runner.merge().await.expect("merge");
15773
15774 assert!(
15775 runner.state.merge.is_none(),
15776 "an empty gate must never be read as a passing one: {:?}",
15777 runner.state.merge
15778 );
15779 }
15780
15781 #[tokio::test]
15788 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15789 let tmp = tempfile::tempdir().expect("tempdir");
15790 let repo = tmp.path().join("repo");
15791 std::fs::create_dir_all(&repo).unwrap();
15792 init_repo(&repo);
15793
15794 let config = Config::default();
15796
15797 let mut state = RunState::new(
15798 repo.clone(),
15799 "main".to_owned(),
15800 "deadbeef".to_owned(),
15801 "task".to_owned(),
15802 config,
15803 );
15804 state.candidates = vec![Candidate {
15805 index: 0,
15806 label: 'A',
15807 agent: "alpha".to_owned(),
15808 branch: "does-not-exist".to_owned(),
15809 worktree: repo.clone(),
15810 summary: String::new(),
15811 stat: String::new(),
15812 files: 0,
15813 commits: 0,
15814 empty: false,
15815 failed: None,
15816 verified_noop: None,
15817 duration_ms: 0,
15818 folded: false,
15819 }];
15820 state.tally = Some(Tally {
15821 first_choice: BTreeMap::from([('A', 1)]),
15822 borda: BTreeMap::new(),
15823 winner: 'A',
15824 rankings: 1,
15825 unanimous_initial: true,
15826 deliberated: false,
15827 changed_votes: 0,
15828 unanimous_final: true,
15829 tie_break: None,
15830 judges: 0,
15831 present: 0,
15832 quorum: 0,
15833 met_quorum: true,
15834 uncontested: Some("only candidate A produced a change".to_owned()),
15835 });
15836 state.reviews = vec![ReviewRound {
15837 round: 1,
15838 head: "deadbeef".to_owned(),
15839 verified_head: None,
15840 verified_at: None,
15841 reviews: Vec::new(),
15842 e2e: Vec::new(),
15843 fix: None,
15844 blocking: 0,
15845 answered: 0,
15846 expected: 0,
15847 clean: true,
15848 verify_retried: false,
15849 e2e_deferred: false,
15850 e2e_defer_reason: None,
15851 progressed: false,
15852 vote_split: false,
15853 reconsideration: Vec::new(),
15854 verdict: None,
15855 }];
15856
15857 let mut runner = Runner {
15858 state,
15859 roles: ResolvedRoles {
15860 implementers: Vec::new(),
15861 judges: Vec::new(),
15862 reviewers: Vec::new(),
15863 fixer: None,
15864 conductor: conductor(),
15865 implementer_roster: Vec::new(),
15866 judge_roster: Vec::new(),
15867 reviewer_roster: Vec::new(),
15868 },
15869 sem: Arc::new(Semaphore::new(1)),
15870 pause: Pause::new(),
15871 interrupt: Pause::new(),
15872 };
15873
15874 runner.gate().await.expect("gate");
15875 assert!(
15876 runner.state.gate_ran,
15877 "zero configured commands is still a real attempt, not an unrun gate"
15878 );
15879 assert!(runner.state.gate.is_empty());
15880 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15881 assert_ne!(
15882 runner.state.status,
15883 RunStatus::Blocked,
15884 "a gate with nothing to check must not read as failed"
15885 );
15886
15887 runner.merge().await.expect("merge");
15888 assert_eq!(
15889 runner.state.status,
15890 RunStatus::Ready,
15891 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15892 );
15893 }
15894
15895 #[tokio::test]
15906 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15907 crate::run::pin_test_home();
15908 let home = crate::run::home();
15909
15910 let tmp = tempfile::tempdir().expect("tempdir");
15911 let repo = tmp.path().join("repo");
15912 std::fs::create_dir_all(&repo).unwrap();
15913 init_repo(&repo);
15914 let cache_dir = tmp.path().join("target");
15917
15918 let mut config = Config::default();
15919 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15920 config.graph.timeout_verify = Some(2);
15923 config.graph.timeout_resource_wait = 2;
15924
15925 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15926 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15927 .expect("no io error acquiring directly")
15928 {
15929 crate::cache::AcquireOutcome::Acquired(g) => g,
15930 crate::cache::AcquireOutcome::Busy(b) => {
15931 panic!("expected the direct acquire to win the lease first: {b:?}")
15932 }
15933 };
15934
15935 let mut state = RunState::new(
15936 repo.clone(),
15937 "main".to_owned(),
15938 "deadbeef".to_owned(),
15939 "task".to_owned(),
15940 config,
15941 );
15942 state.candidates = vec![Candidate {
15943 index: 0,
15944 label: 'A',
15945 agent: "alpha".to_owned(),
15946 branch: "does-not-exist".to_owned(),
15947 worktree: repo.clone(),
15948 summary: String::new(),
15949 stat: String::new(),
15950 files: 0,
15951 commits: 0,
15952 empty: false,
15953 failed: None,
15954 verified_noop: None,
15955 duration_ms: 0,
15956 folded: false,
15957 }];
15958 state.tally = Some(Tally {
15959 first_choice: BTreeMap::from([('A', 1)]),
15960 borda: BTreeMap::new(),
15961 winner: 'A',
15962 rankings: 1,
15963 unanimous_initial: true,
15964 deliberated: false,
15965 changed_votes: 0,
15966 unanimous_final: true,
15967 tie_break: None,
15968 judges: 0,
15969 present: 0,
15970 quorum: 0,
15971 met_quorum: true,
15972 uncontested: Some("only candidate A produced a change".to_owned()),
15973 });
15974 state.reviews = vec![ReviewRound {
15975 round: 1,
15976 head: "deadbeef".to_owned(),
15977 verified_head: None,
15978 verified_at: None,
15979 reviews: Vec::new(),
15980 e2e: Vec::new(),
15981 fix: None,
15982 blocking: 0,
15983 answered: 0,
15984 expected: 0,
15985 clean: true,
15986 verify_retried: false,
15987 e2e_deferred: false,
15988 e2e_defer_reason: None,
15989 progressed: false,
15990 vote_split: false,
15991 reconsideration: Vec::new(),
15992 verdict: None,
15993 }];
15994
15995 let mut runner = Runner {
15996 state,
15997 roles: ResolvedRoles {
15998 implementers: Vec::new(),
15999 judges: Vec::new(),
16000 reviewers: Vec::new(),
16001 fixer: None,
16002 conductor: conductor(),
16003 implementer_roster: Vec::new(),
16004 judge_roster: Vec::new(),
16005 reviewer_roster: Vec::new(),
16006 },
16007 sem: Arc::new(Semaphore::new(1)),
16008 pause: Pause::new(),
16009 interrupt: Pause::new(),
16010 };
16011
16012 let started = std::time::Instant::now();
16013 runner.gate().await.expect("gate");
16014 assert!(
16015 started.elapsed() < Duration::from_secs(1),
16016 "a gate with nothing to run must never wait on a lease it never needed"
16017 );
16018 assert!(
16019 runner.state.gate_ran,
16020 "zero commands is still a real, immediate attempt"
16021 );
16022 assert!(runner.state.gate.is_empty());
16023 assert_ne!(
16024 runner.state.status,
16025 RunStatus::Blocked,
16026 "must not read as resource-blocked on a lease it never asked for"
16027 );
16028 }
16029
16030 #[cfg(unix)]
16031 #[tokio::test]
16032 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16033 crate::run::pin_test_home();
16034 let tmp = tempfile::tempdir().unwrap();
16035 let mut config = Config::default();
16036 config.disk.min_free_bytes = 0;
16037 let mut state = RunState::new(
16038 tmp.path().to_owned(),
16039 "main".into(),
16040 "base".into(),
16041 "verification evidence".into(),
16042 config,
16043 );
16044 let (outcomes, pids) = run_commands(
16045 &mut state,
16046 "gate",
16047 "gate",
16048 1,
16049 &["sh".into(), "-c".into()],
16050 &[
16051 "printf before-timeout; exec sleep 30".into(),
16052 "kill -TERM $$".into(),
16053 "exit 7".into(),
16054 "printf success".into(),
16055 ],
16056 tmp.path(),
16057 Duration::from_secs(1),
16058 )
16059 .await;
16060 wait_for_timed_out_children_to_die(&pids).await;
16061 assert_eq!(
16062 pids.len(),
16063 1,
16064 "unleased timeout retains its death-check PID"
16065 );
16066 assert_eq!(outcomes.len(), 4);
16067 assert_eq!(outcomes[0].code, None);
16068 assert!(outcomes[0].output_tail.contains("before-timeout"));
16069 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16070 assert_eq!(outcomes[1].code, None);
16071 assert!(outcomes[1].output_tail.contains("command terminated:"));
16072 assert!(!outcomes[1].output_tail.contains("timed out"));
16073 assert_eq!(outcomes[2].code, Some(7));
16074 assert!(!outcomes[2].output_tail.contains("timed out"));
16075 assert_eq!(outcomes[3].code, Some(0));
16076 assert_eq!(outcomes[3].output_tail, "success");
16077 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16078 }
16079
16080 #[tokio::test]
16090 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16091 crate::run::pin_test_home();
16092
16093 let tmp = tempfile::tempdir().expect("tempdir");
16094 let repo = tmp.path().join("repo");
16095 std::fs::create_dir_all(&repo).unwrap();
16096 init_repo(&repo);
16097
16098 let mut config = Config::default();
16099 config.verify.gate = vec![
16100 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16101 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16102 .to_owned(),
16103 ];
16104
16105 let mut state = RunState::new(
16106 repo.clone(),
16107 "main".to_owned(),
16108 "deadbeef".to_owned(),
16109 "task".to_owned(),
16110 config,
16111 );
16112 let run_id = state.id.clone();
16113 state.candidates = vec![Candidate {
16114 index: 0,
16115 label: 'A',
16116 agent: "alpha".to_owned(),
16117 branch: "does-not-exist".to_owned(),
16118 worktree: repo.clone(),
16119 summary: String::new(),
16120 stat: String::new(),
16121 files: 0,
16122 commits: 0,
16123 empty: false,
16124 failed: None,
16125 verified_noop: None,
16126 duration_ms: 0,
16127 folded: false,
16128 }];
16129 state.tally = Some(Tally {
16130 first_choice: BTreeMap::from([('A', 1)]),
16131 borda: BTreeMap::new(),
16132 winner: 'A',
16133 rankings: 1,
16134 unanimous_initial: true,
16135 deliberated: false,
16136 changed_votes: 0,
16137 unanimous_final: true,
16138 tie_break: None,
16139 judges: 0,
16140 present: 0,
16141 quorum: 0,
16142 met_quorum: true,
16143 uncontested: Some("only candidate A produced a change".to_owned()),
16144 });
16145 state.reviews = vec![ReviewRound {
16146 round: 1,
16147 head: "deadbeef".to_owned(),
16148 verified_head: None,
16149 verified_at: None,
16150 reviews: Vec::new(),
16151 e2e: Vec::new(),
16152 fix: None,
16153 blocking: 0,
16154 answered: 0,
16155 expected: 0,
16156 clean: true,
16157 verify_retried: false,
16158 e2e_deferred: false,
16159 e2e_defer_reason: None,
16160 progressed: false,
16161 vote_split: false,
16162 reconsideration: Vec::new(),
16163 verdict: None,
16164 }];
16165
16166 let mut runner = Runner {
16167 state,
16168 roles: ResolvedRoles {
16169 implementers: Vec::new(),
16170 judges: Vec::new(),
16171 reviewers: Vec::new(),
16172 fixer: None,
16173 conductor: conductor(),
16174 implementer_roster: Vec::new(),
16175 judge_roster: Vec::new(),
16176 reviewer_roster: Vec::new(),
16177 },
16178 sem: Arc::new(Semaphore::new(1)),
16179 pause: Pause::new(),
16180 interrupt: Pause::new(),
16181 };
16182
16183 let started_marker = repo.join("started.marker");
16184 let release_marker = repo.join("release.marker");
16185 let poller = tokio::spawn(async move {
16186 for _ in 0..100 {
16191 if started_marker.exists()
16192 && let Ok(s) = crate::run::RunState::load(&run_id)
16193 && let Some(a) = s.active.get("gate")
16194 {
16195 std::fs::write(&release_marker, b"go").expect("release marker");
16196 return Some(a.clone());
16197 }
16198 tokio::time::sleep(Duration::from_millis(50)).await;
16199 }
16200 None
16201 });
16202
16203 runner.gate().await.expect("gate");
16204 let captured = poller.await.expect("poller task");
16205 let captured = captured.expect(
16206 "the poller never saw a `gate` task entry in run.json while the command was \
16207 still blocked on its own release marker",
16208 );
16209
16210 assert_eq!(captured.task.as_deref(), Some("gate"));
16211 assert_eq!(captured.node, "gate");
16212 assert_eq!(captured.index, Some(1));
16213 assert_eq!(captured.total, Some(1));
16214 assert!(
16215 captured
16216 .command
16217 .as_deref()
16218 .is_some_and(|c| c.contains("started.marker")),
16219 "{captured:?}"
16220 );
16221
16222 assert!(
16223 runner.state.active.is_empty(),
16224 "the entry must be cleared once the command actually finished: {:?}",
16225 runner.state.active
16226 );
16227 assert!(runner.state.gate_ran);
16228 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16229 }
16230
16231 #[tokio::test]
16234 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16235 use crate::verdict::{Finding, ReviewVote, Severity};
16236 crate::run::pin_test_home();
16237 let tmp = tempfile::tempdir().expect("tempdir");
16238 let repo = tmp.path().join("repo");
16239 std::fs::create_dir_all(&repo).unwrap();
16240 init_repo(&repo);
16241
16242 for (severity, vote, expect) in [
16243 (Severity::Major, ReviewVote::Reject, true),
16244 (Severity::Minor, ReviewVote::Reject, false),
16245 (Severity::Major, ReviewVote::Approve, false),
16246 ] {
16247 let mut round = review_round(false, 1, 1, 1, false, true);
16248 round.reviews = vec![ReviewRecord {
16249 reviewer: 1,
16250 agent: "alpha".to_owned(),
16251 summary: String::new(),
16252 findings: vec![Finding {
16253 id: "R1-1-1".to_owned(),
16254 severity,
16255 file: None,
16256 line: None,
16257 title: "t".to_owned(),
16258 detail: String::new(),
16259 }],
16260 vote: Some(vote),
16261 failed: None,
16262 duration_ms: 0,
16263 attempts: 0,
16264 }];
16265 let mut state = RunState::new(
16266 repo.clone(),
16267 "main".to_owned(),
16268 "deadbeef".to_owned(),
16269 "task".to_owned(),
16270 Config::default(),
16271 );
16272 state.reviews = vec![round];
16273 let mut runner = Runner {
16274 state,
16275 roles: ResolvedRoles {
16276 implementers: Vec::new(),
16277 judges: Vec::new(),
16278 reviewers: Vec::new(),
16279 fixer: None,
16280 conductor: conductor(),
16281 implementer_roster: Vec::new(),
16282 judge_roster: Vec::new(),
16283 reviewer_roster: Vec::new(),
16284 },
16285 sem: Arc::new(Semaphore::new(1)),
16286 pause: Pause::new(),
16287 interrupt: Pause::new(),
16288 };
16289 let shell = runner.state.config.shell();
16290 runner
16291 .stop_reviewing("round budget spent", &shell, &repo)
16292 .await
16293 .expect("stop_reviewing");
16294 assert_eq!(runner.state.status, RunStatus::Gating);
16295 assert_eq!(
16296 runner.state.contested_handoff.is_some(),
16297 expect,
16298 "{severity:?} + {vote:?}"
16299 );
16300 }
16301 }
16302
16303 #[tokio::test]
16316 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16317 crate::run::pin_test_home();
16318 let home = crate::run::home();
16319
16320 let tmp = tempfile::tempdir().expect("tempdir");
16321 let repo = tmp.path().join("repo");
16322 std::fs::create_dir_all(&repo).unwrap();
16323 init_repo(&repo);
16324 let head = crate::git::rev_parse(&repo, "HEAD")
16325 .await
16326 .expect("rev-parse");
16327 let cache_dir = tmp.path().join("target");
16330
16331 let mut config = Config::default();
16332 config.verify.e2e = vec![format!(
16333 "CARGO_TARGET_DIR='{}' test -f README.md",
16334 cache_dir.display()
16335 )];
16336 config.graph.review_rounds = 1;
16337 config.graph.timeout_verify = Some(2);
16340 config.graph.timeout_resource_wait = 2;
16341
16342 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16343 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16344 .expect("no io error acquiring directly")
16345 {
16346 crate::cache::AcquireOutcome::Acquired(g) => g,
16347 crate::cache::AcquireOutcome::Busy(b) => {
16348 panic!("expected the direct acquire to win the lease first: {b:?}")
16349 }
16350 };
16351
16352 let mut state = RunState::new(
16353 repo.clone(),
16354 "main".to_owned(),
16355 head.clone(),
16356 "task".to_owned(),
16357 config,
16358 );
16359 state.candidates = vec![Candidate {
16360 index: 0,
16361 label: 'A',
16362 agent: "alpha".to_owned(),
16363 branch: "does-not-exist".to_owned(),
16364 worktree: repo.clone(),
16365 summary: String::new(),
16366 stat: String::new(),
16367 files: 0,
16368 commits: 0,
16369 empty: false,
16370 failed: None,
16371 verified_noop: None,
16372 duration_ms: 0,
16373 folded: false,
16374 }];
16375 state.tally = Some(Tally {
16376 first_choice: BTreeMap::from([('A', 1)]),
16377 borda: BTreeMap::new(),
16378 winner: 'A',
16379 rankings: 1,
16380 unanimous_initial: true,
16381 deliberated: false,
16382 changed_votes: 0,
16383 unanimous_final: true,
16384 tie_break: None,
16385 judges: 0,
16386 present: 0,
16387 quorum: 0,
16388 met_quorum: true,
16389 uncontested: Some("only candidate A produced a change".to_owned()),
16390 });
16391 state.reviews = vec![ReviewRound {
16395 round: 1,
16396 head: head.clone(),
16397 verified_head: None,
16398 verified_at: None,
16399 reviews: Vec::new(),
16400 e2e: Vec::new(),
16401 fix: None,
16402 blocking: 1,
16403 answered: 1,
16404 expected: 1,
16405 clean: false,
16406 verify_retried: false,
16407 e2e_deferred: true,
16408 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16409 progressed: false,
16410 vote_split: false,
16411 reconsideration: Vec::new(),
16412 verdict: None,
16413 }];
16414
16415 let mut runner = Runner {
16416 state,
16417 roles: ResolvedRoles {
16418 implementers: Vec::new(),
16419 judges: Vec::new(),
16420 reviewers: Vec::new(),
16421 fixer: None,
16422 conductor: conductor(),
16423 implementer_roster: Vec::new(),
16424 judge_roster: Vec::new(),
16425 reviewer_roster: Vec::new(),
16426 },
16427 sem: Arc::new(Semaphore::new(1)),
16428 pause: Pause::new(),
16429 interrupt: Pause::new(),
16430 };
16431
16432 let shell = runner.state.config.shell();
16433 runner
16434 .stop_reviewing("round budget spent", &shell, &repo)
16435 .await
16436 .expect("stop_reviewing");
16437
16438 let last = runner.state.reviews.last().expect("round record");
16439 assert_eq!(
16440 last.e2e_status(),
16441 E2eStatus::ResourceBlocked,
16442 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16443 failed: {last:?}"
16444 );
16445 assert_eq!(
16446 last.verified_head.as_deref(),
16447 Some(head.as_str()),
16448 "which commit this attempt targeted is known even though nothing finished checking \
16449 it"
16450 );
16451 let first_attempt_at = last
16452 .verified_at
16453 .expect("when this attempt ran is known too");
16454 assert_ne!(
16455 runner.state.status,
16456 RunStatus::Blocked,
16457 "contention is evidence about the machine, not the patch — it must not settle the \
16458 run as blocked: {:?}",
16459 runner.state.status
16460 );
16461 assert!(
16462 !runner
16463 .state
16464 .events
16465 .iter()
16466 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16467 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16468 runner.state.events
16469 );
16470
16471 runner
16476 .stop_reviewing("round budget spent", &shell, &repo)
16477 .await
16478 .expect("stop_reviewing retry");
16479 assert_eq!(
16480 runner.state.reviews.len(),
16481 1,
16482 "no new round was started: {:?}",
16483 runner.state.reviews
16484 );
16485 let last = runner.state.reviews.last().expect("round record");
16486 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16487 assert!(
16488 last.verified_at.expect("still known") > first_attempt_at,
16489 "a second reentry must be a fresh attempt, not a stale copy of the first"
16490 );
16491 assert_ne!(runner.state.status, RunStatus::Blocked);
16492
16493 held.release();
16494 }
16495
16496 #[cfg(unix)]
16497 #[tokio::test]
16498 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
16499 ask_test_home();
16500 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
16501 let tmp = tempfile::tempdir().unwrap();
16502 let repo = tmp.path().join("repo");
16503 std::fs::create_dir(&repo).unwrap();
16504 init_repo(&repo);
16505 let run_git = |args: &[&str]| {
16506 let out = std::process::Command::new("git")
16507 .args(args)
16508 .current_dir(&repo)
16509 .output()
16510 .unwrap();
16511 assert!(
16512 out.status.success(),
16513 "{args:?}: {}",
16514 String::from_utf8_lossy(&out.stderr)
16515 );
16516 };
16517 let origin = tmp.path().join("origin.git");
16518 run_git(&["init", "--bare", origin.to_str().unwrap()]);
16519 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
16520 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
16521 run_git(&["add", "spec.rs"]);
16522 run_git(&["commit", "-m", "initial spec"]);
16523 run_git(&["push", "origin", "main"]);
16524 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16525 run_git(&["checkout", "-b", "fix/review"]);
16526 std::fs::write(
16527 repo.join("check.rs"),
16528 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
16529 )
16530 .unwrap();
16531 run_git(&["add", "check.rs"]);
16532 run_git(&["commit", "-m", "use spec"]);
16533 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
16534 run_git(&["checkout", "main"]);
16535 std::fs::write(
16536 repo.join("spec.rs"),
16537 "struct AgentSpec { max_turn_secs: u64 }\n",
16538 )
16539 .unwrap();
16540 run_git(&["add", "spec.rs"]);
16541 run_git(&["commit", "-m", "extend spec"]);
16542 run_git(&["push", "origin", "main"]);
16543 run_git(&["checkout", "fix/review"]);
16544
16545 let mut runner = runner_at(RunStatus::Reviewing);
16546 runner.state.repo = repo.clone();
16547 runner.state.base_commit = base;
16548 runner.state.pr_head = Some("fix/review".into());
16549 runner.state.config.graph.review_rounds = max_rounds;
16550 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16551 runner.state.config.graph.sessions = false;
16552 runner.state.config.disk.min_free_bytes = 0;
16553 runner.state.config.verify.e2e =
16554 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
16555 runner.state.candidates = vec![Candidate {
16556 index: 0,
16557 label: 'A',
16558 agent: "fixture".into(),
16559 branch: "fix/review".into(),
16560 worktree: repo.clone(),
16561 summary: String::new(),
16562 stat: String::new(),
16563 files: 1,
16564 commits: 1,
16565 empty: false,
16566 failed: None,
16567 verified_noop: None,
16568 duration_ms: 0,
16569 folded: false,
16570 }];
16571 runner.state.tally = Some(Tally {
16572 first_choice: BTreeMap::new(),
16573 borda: BTreeMap::new(),
16574 winner: 'A',
16575 rankings: 0,
16576 unanimous_initial: true,
16577 deliberated: false,
16578 changed_votes: 0,
16579 unanimous_final: true,
16580 tie_break: None,
16581 judges: 0,
16582 present: 0,
16583 quorum: 0,
16584 met_quorum: true,
16585 uncontested: Some("review-only".into()),
16586 });
16587 let mut last = review_round(false, 0, 1, 1, false, false);
16588 last.head = reviewed.clone();
16589 last.e2e_deferred = true;
16590 last.e2e[0].resource_blocked = true;
16591 last.e2e[0].output_tail = "waiting for the shared build cache".into();
16592 if max_rounds == 1 {
16593 last.e2e.clear();
16594 }
16595 runner.state.reviews = vec![last];
16596 let calls = tmp.path().join("fix-calls");
16597 let mut fixer = spec("repair");
16598 fixer
16599 .env
16600 .insert("FIX_CALLS".into(), calls.display().to_string());
16601 fixer.command = vec!["sh".into(), "-c".into(), r#"
16602 echo fix >> "$FIX_CALLS"
16603 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
16604 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
16605 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
16606 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
16607 "#.into()];
16608 let mut reviewer = spec("reviewer");
16609 reviewer.command = vec![
16610 "sh".into(),
16611 "-c".into(),
16612 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
16613 .into(),
16614 ];
16615 runner.roles.fixer = Some(vec![fixer]);
16616 runner.roles.reviewers = vec![reviewer];
16617
16618 runner.sync_to_base().await.unwrap();
16619 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
16620 assert_ne!(rebased, reviewed);
16621 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
16622 if reentry {
16623 let shell = runner.state.config.shell();
16626 runner
16627 .stop_unresolved_review("deferred verification", &shell, &repo)
16628 .await
16629 .unwrap();
16630 assert!(runner.catchup_repairable());
16631 assert!(!calls.exists());
16632 runner.state = RunState::load(&runner.state.id).unwrap();
16633 assert!(runner.catchup_repairable());
16634 }
16635 runner.review_loop().await.unwrap();
16636 let first = &runner.state.reviews[0];
16637 assert_eq!(first.head, reviewed);
16638 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
16639 assert!(
16640 first.verify_retried,
16641 "a reproduced build failure gets exactly one retry: {:?}",
16642 first.e2e
16643 );
16644 assert!(first.e2e[0].output_tail.contains("E0063"));
16645 assert!(!runner.catchup_repairable());
16646 if max_rounds == 2 {
16647 assert_eq!(
16648 runner.state.status,
16649 RunStatus::Gating,
16650 "{}",
16651 crate::report::run(&runner.state)
16652 );
16653 assert_eq!(runner.state.reviews.len(), 2);
16654 assert!(first.fix.as_ref().unwrap().committed);
16655 assert!(runner.state.reviews[1].clean);
16656 assert_eq!(
16657 runner
16658 .state
16659 .reviews
16660 .iter()
16661 .filter(|r| r.fix.is_some())
16662 .count(),
16663 1
16664 );
16665 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
16666 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
16667 } else {
16668 assert_eq!(runner.state.status, RunStatus::Blocked);
16669 assert!(first.fix.is_none());
16670 assert!(!calls.exists());
16671 let recovery = crate::recovery::assess(
16672 "prior",
16673 Some(&runner.state),
16674 &runner.landing_base(),
16675 crate::recovery::Relation::Diverged,
16676 );
16677 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
16678 let reasons = recovery.reasons.join("\n");
16679 assert!(
16680 reasons.contains(&reviewed) && reasons.contains(&rebased),
16681 "{reasons}"
16682 );
16683 assert!(
16684 reasons.contains("base sync to")
16685 && reasons.contains("additional review is required"),
16686 "{reasons}"
16687 );
16688 }
16689 }
16690 }
16691
16692 #[tokio::test]
16704 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16705 crate::run::pin_test_home();
16706 let home = crate::run::home();
16707
16708 let tmp = tempfile::tempdir().expect("tempdir");
16709 let repo = tmp.path().join("repo");
16710 std::fs::create_dir_all(&repo).unwrap();
16711 init_repo(&repo);
16712 let head = crate::git::rev_parse(&repo, "HEAD")
16713 .await
16714 .expect("rev-parse");
16715 let cache_dir = tmp.path().join("target");
16716
16717 let mut config = Config::default();
16718 config.verify.e2e = vec![format!(
16719 "CARGO_TARGET_DIR='{}' test -f README.md",
16720 cache_dir.display()
16721 )];
16722 config.graph.review_rounds = 1;
16723 config.graph.timeout_verify = Some(2);
16724 config.graph.timeout_resource_wait = 2;
16725
16726 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16727 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16728 .expect("no io error acquiring directly")
16729 {
16730 crate::cache::AcquireOutcome::Acquired(g) => g,
16731 crate::cache::AcquireOutcome::Busy(b) => {
16732 panic!("expected the direct acquire to win the lease first: {b:?}")
16733 }
16734 };
16735
16736 let mut state = RunState::new(
16737 repo.clone(),
16738 "main".to_owned(),
16739 head.clone(),
16740 "task".to_owned(),
16741 config,
16742 );
16743 state.candidates = vec![Candidate {
16744 index: 0,
16745 label: 'A',
16746 agent: "alpha".to_owned(),
16747 branch: "does-not-exist".to_owned(),
16748 worktree: repo.clone(),
16749 summary: String::new(),
16750 stat: String::new(),
16751 files: 0,
16752 commits: 0,
16753 empty: false,
16754 failed: None,
16755 verified_noop: None,
16756 duration_ms: 0,
16757 folded: false,
16758 }];
16759 state.tally = Some(Tally {
16760 first_choice: BTreeMap::from([('A', 1)]),
16761 borda: BTreeMap::new(),
16762 winner: 'A',
16763 rankings: 1,
16764 unanimous_initial: true,
16765 deliberated: false,
16766 changed_votes: 0,
16767 unanimous_final: true,
16768 tie_break: None,
16769 judges: 0,
16770 present: 0,
16771 quorum: 0,
16772 met_quorum: true,
16773 uncontested: Some("only candidate A produced a change".to_owned()),
16774 });
16775 state.reviews = vec![ReviewRound {
16779 round: 1,
16780 head: head.clone(),
16781 verified_head: Some(head.clone()),
16782 verified_at: Some(jiff::Timestamp::now()),
16783 reviews: Vec::new(),
16784 e2e: vec![CommandOutcome {
16785 command: format!(
16786 "CARGO_TARGET_DIR='{}' test -f README.md",
16787 cache_dir.display()
16788 ),
16789 code: None,
16790 output_tail: "waiting for the shared build cache".to_owned(),
16791 duration_ms: 0,
16792 resource_blocked: true,
16793 }],
16794 fix: None,
16795 blocking: 1,
16796 answered: 1,
16797 expected: 1,
16798 clean: false,
16799 verify_retried: false,
16800 e2e_deferred: false,
16801 e2e_defer_reason: None,
16802 progressed: false,
16803 vote_split: false,
16804 reconsideration: Vec::new(),
16805 verdict: None,
16806 }];
16807
16808 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16809 let mut runner = Runner {
16810 state,
16811 roles: ResolvedRoles {
16812 implementers: Vec::new(),
16813 judges: Vec::new(),
16814 reviewers: Vec::new(),
16815 fixer: None,
16816 conductor: conductor(),
16817 implementer_roster: Vec::new(),
16818 judge_roster: Vec::new(),
16819 reviewer_roster: Vec::new(),
16820 },
16821 sem: Arc::new(Semaphore::new(1)),
16822 pause: Pause::new(),
16823 interrupt: Pause::new(),
16824 };
16825
16826 runner.review_loop().await.expect("review_loop");
16831
16832 assert_eq!(
16833 runner.state.reviews.len(),
16834 1,
16835 "no new round was started on top of the unresolved one: {:?}",
16836 runner.state.reviews
16837 );
16838 let last = &runner.state.reviews[0];
16839 assert_eq!(
16840 last.e2e_status(),
16841 E2eStatus::ResourceBlocked,
16842 "still contended: {last:?}"
16843 );
16844 assert!(
16845 last.verified_at.expect("still known") > first_attempt_at,
16846 "review_loop must have actually retried the check, not left it exactly as found"
16847 );
16848 assert_ne!(
16849 runner.state.status,
16850 RunStatus::Blocked,
16851 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16852 runner.state.status
16853 );
16854
16855 held.release();
16856 }
16857
16858 #[tokio::test]
16859 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16860 crate::run::pin_test_home();
16861 for status in [RunStatus::Blocked, RunStatus::Gating] {
16862 let mut runner = runner_at(status);
16863 runner.state.config.graph.land = true;
16864 runner.state.gate_ran = true;
16865 runner.state.merge = Some(MergeOutcome {
16866 mode: MergeMode::Pr,
16867 ok: false,
16868 detail: "the fixer produced no commit".into(),
16869 empty: false,
16870 });
16871 let url = "https://example.invalid/o/r/pull/1";
16872 runner.state.pr = Some(crate::run::PrRecord {
16873 url: url.into(),
16874 number: 1,
16875 state: "open".into(),
16876 checks: "red".into(),
16877 round: 1,
16878 rounds: 1,
16879 red_at_merge: Vec::new(),
16880 });
16881 runner.state.save().unwrap();
16882 runner.execute_graph().await.unwrap();
16886 assert_eq!(runner.state.status, RunStatus::Blocked);
16887 assert!(runner.state.land_sync_pending);
16888 assert!(runner.state.events.iter().any(|event|
16889 event.node == "land" && event.message == format!("watching {url}")));
16890 assert_eq!(
16891 runner.state.merge.as_ref().unwrap().detail,
16892 "the fixer produced no commit"
16893 );
16894 }
16895 }
16896
16897 #[tokio::test]
16898 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16899 crate::run::pin_test_home();
16900 let tmp = tempfile::tempdir().expect("tempdir");
16901 let repo = tmp.path().join("repo");
16902 std::fs::create_dir_all(&repo).unwrap();
16903 init_repo(&repo);
16904
16905 let mut config = Config::default();
16906 config.merge.mode = MergeMode::Pr;
16907 config.graph.land = true;
16908 config.graph.land_approval = false;
16909
16910 let mut state = RunState::new(
16911 repo.clone(),
16912 "main".to_owned(),
16913 "deadbeef".to_owned(),
16914 "task".to_owned(),
16915 config,
16916 );
16917 state.candidates = vec![Candidate {
16918 index: 0,
16919 label: 'A',
16920 agent: "alpha".to_owned(),
16921 branch: "does-not-exist".to_owned(),
16922 worktree: repo.clone(),
16923 summary: String::new(),
16924 stat: String::new(),
16925 files: 0,
16926 commits: 0,
16927 empty: false,
16928 failed: None,
16929 verified_noop: None,
16930 duration_ms: 0,
16931 folded: false,
16932 }];
16933 state.tally = Some(Tally {
16934 first_choice: BTreeMap::from([('A', 1)]),
16935 borda: BTreeMap::new(),
16936 winner: 'A',
16937 rankings: 1,
16938 unanimous_initial: true,
16939 deliberated: false,
16940 changed_votes: 0,
16941 unanimous_final: true,
16942 tie_break: None,
16943 judges: 0,
16944 present: 0,
16945 quorum: 0,
16946 met_quorum: true,
16947 uncontested: Some("only candidate A produced a change".to_owned()),
16948 });
16949 state.reviews = vec![ReviewRound {
16950 round: 1,
16951 head: "deadbeef".to_owned(),
16952 verified_head: None,
16953 verified_at: None,
16954 reviews: Vec::new(),
16955 e2e: Vec::new(),
16956 fix: None,
16957 blocking: 0,
16958 answered: 0,
16959 expected: 0,
16960 clean: true,
16961 verify_retried: false,
16962 e2e_deferred: false,
16963 e2e_defer_reason: None,
16964 progressed: false,
16965 vote_split: false,
16966 reconsideration: Vec::new(),
16967 verdict: None,
16968 }];
16969 state.gate = vec![CommandOutcome {
16970 command: "test".to_owned(),
16971 code: Some(0),
16972 output_tail: String::new(),
16973 duration_ms: 0,
16974 resource_blocked: false,
16975 }];
16976 state.gate_ran = true;
16977 state.status = RunStatus::Landing;
16981 state.merge = Some(MergeOutcome {
16982 mode: MergeMode::Pr,
16983 ok: true,
16984 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16985 empty: false,
16986 });
16987
16988 ask_test_home();
16992 state.save().expect("save landing run");
16995 let store = ask::Questions::open();
16996 let q = ask_open_question(&store, &state.id);
16997
16998 let mut runner = Runner {
16999 state,
17000 roles: ResolvedRoles {
17001 implementers: Vec::new(),
17002 judges: Vec::new(),
17003 reviewers: Vec::new(),
17004 fixer: None,
17005 conductor: conductor(),
17006 implementer_roster: Vec::new(),
17007 judge_roster: Vec::new(),
17008 reviewer_roster: Vec::new(),
17009 },
17010 sem: Arc::new(Semaphore::new(1)),
17011 pause: Pause::new(),
17012 interrupt: Pause::new(),
17013 };
17014
17015 runner.execute().await.expect("execute");
17020
17021 assert_eq!(
17022 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17023 Some("https://example.invalid/x/y/pull/1"),
17024 "reentry must not push again or open a second pull request over the \
17025 one `land` is already watching"
17026 );
17027 assert_ne!(
17028 runner.state.status,
17029 RunStatus::Landing,
17030 "land could not actually reach the fake pull request, so it must \
17031 have given up rather than left the run silently parked forever"
17032 );
17033 assert_eq!(runner.state.status, RunStatus::Blocked);
17037 assert!(
17038 store.get(&q.id).unwrap().status.open(),
17039 "Blocked is still alive; settle_questions must have been a no-op here"
17040 );
17041 }
17042
17043 fn state_with_round(round: ReviewRound) -> RunState {
17044 let mut s = RunState::new(
17045 PathBuf::from("/repo"),
17046 "main".to_owned(),
17047 "abc1234".to_owned(),
17048 "add retries".to_owned(),
17049 Config::default(),
17050 );
17051 s.reviews = vec![round];
17052 s
17053 }
17054
17055 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17056 crate::verdict::Finding {
17057 id: id.to_owned(),
17058 severity,
17059 file: None,
17060 line: None,
17061 title: title.to_owned(),
17062 detail: String::new(),
17063 }
17064 }
17065
17066 #[test]
17067 fn pr_body_names_open_findings_and_declined_ones() {
17068 let round = ReviewRound {
17069 round: 2,
17070 head: "deadbee".to_owned(),
17071 verified_head: None,
17072 verified_at: None,
17073 reviews: vec![ReviewRecord {
17074 attempts: 0,
17075 reviewer: 1,
17076 agent: "alpha".to_owned(),
17077 summary: String::new(),
17078 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17079 vote: None,
17080 failed: None,
17081 duration_ms: 0,
17082 }],
17083 e2e: vec![CommandOutcome {
17084 command: "cargo test".to_owned(),
17085 code: Some(0),
17086 output_tail: String::new(),
17087 duration_ms: 0,
17088 resource_blocked: false,
17089 }],
17090 verify_retried: false,
17091 e2e_deferred: false,
17092 e2e_defer_reason: None,
17093 fix: Some(FixRecord {
17094 agent: "alpha".to_owned(),
17095 addressed: Vec::new(),
17096 rejected: vec![crate::verdict::Rejection {
17097 id: "R1-1-1".to_owned(),
17098 why: "not reachable from any caller".to_owned(),
17099 }],
17100 notes: String::new(),
17101 committed: true,
17102 failed: None,
17103 duration_ms: 0,
17104 continuation: None,
17105 }),
17106 blocking: 0,
17107 answered: 1,
17108 expected: 1,
17109 clean: false,
17110 progressed: true,
17111 vote_split: false,
17112 reconsideration: Vec::new(),
17113 verdict: None,
17114 };
17115 let state = state_with_round(round);
17116 let body = pr_message(&state, 'A').body;
17117
17118 assert!(body.contains("add retries"), "the task must still be there");
17119 assert!(body.contains("R2-1-1"), "{body}");
17120 assert!(!body.contains("unused import"), "{body}");
17121 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17122 assert!(
17123 !body.contains("not reachable from any caller"),
17124 "the reason it was declined: {body}"
17125 );
17126 }
17127
17128 #[test]
17129 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17130 let round = ReviewRound {
17131 round: 1,
17132 head: "deadbee".to_owned(),
17133 verified_head: None,
17134 verified_at: None,
17135 reviews: vec![ReviewRecord {
17136 attempts: 0,
17137 reviewer: 1,
17138 agent: "alpha".to_owned(),
17139 summary: String::new(),
17140 findings: Vec::new(),
17141 vote: None,
17142 failed: None,
17143 duration_ms: 0,
17144 }],
17145 e2e: Vec::new(),
17146 verify_retried: false,
17147 e2e_deferred: false,
17148 e2e_defer_reason: None,
17149 fix: None,
17150 blocking: 0,
17151 answered: 1,
17152 expected: 1,
17153 clean: true,
17154 progressed: false,
17155 vote_split: false,
17156 reconsideration: Vec::new(),
17157 verdict: None,
17158 };
17159 let state = state_with_round(round);
17160 let body = pr_message(&state, 'A').body;
17161 assert!(!body.contains("Open review findings"), "{body}");
17162 assert!(!body.contains("Declined"), "{body}");
17163 }
17164
17165 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17166 let mut state = RunState::new(
17167 PathBuf::from("/repo"),
17168 "main".to_owned(),
17169 "abc1234".to_owned(),
17170 instruction.to_owned(),
17171 Config::default(),
17172 );
17173 state.candidates.push(Candidate {
17174 index: 0,
17175 label: 'A',
17176 agent: "alpha".to_owned(),
17177 branch: "magi/x/A".to_owned(),
17178 worktree: PathBuf::from("/wt"),
17179 summary: summary.to_owned(),
17180 stat: String::new(),
17181 files: 1,
17182 commits: 1,
17183 empty: false,
17184 failed: None,
17185 verified_noop: None,
17186 folded: false,
17187 duration_ms: 0,
17188 });
17189 state
17190 }
17191
17192 fn review_state(subjects: &[&str]) -> RunState {
17193 let mut state = state_with_summary(
17194 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17195 "",
17196 );
17197 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17198 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17199 state
17200 }
17201
17202 #[tokio::test]
17205 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17206 ask_test_home();
17207 let tmp = tempfile::tempdir().unwrap();
17208 let repo = tmp.path().join("repo");
17209 std::fs::create_dir_all(&repo).unwrap();
17210 init_repo(&repo);
17211 let origin = tmp.path().join("origin.git");
17212 let g = |dir: &Path, args: &[&str]| {
17213 let out = std::process::Command::new("git")
17214 .args(args)
17215 .current_dir(dir)
17216 .quiet()
17217 .output()
17218 .expect("spawn git");
17219 assert!(
17220 out.status.success(),
17221 "git {args:?}: {}",
17222 String::from_utf8_lossy(&out.stderr)
17223 );
17224 };
17225 g(
17226 tmp.path(),
17227 &[
17228 "clone",
17229 "--bare",
17230 "-q",
17231 repo.to_str().unwrap(),
17232 origin.to_str().unwrap(),
17233 ],
17234 );
17235 g(
17236 &repo,
17237 &["remote", "add", "origin", origin.to_str().unwrap()],
17238 );
17239 let c1 = git::rev_parse(&repo, "main").await.unwrap();
17240
17241 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17243 g(&repo, &["add", "-A"]);
17244 g(
17245 &repo,
17246 &["commit", "-q", "-m", "chore(deps): update a crate"],
17247 );
17248 g(&repo, &["push", "-q", "origin", "main"]);
17249 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17250 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17251 g(&repo, &["add", "-A"]);
17252 g(
17253 &repo,
17254 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17255 );
17256 g(&repo, &["checkout", "-q", "main"]);
17257
17258 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17259 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17260 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17262 assert_eq!(fallback, c1);
17263
17264 let mut state = review_state(&[
17265 "chore(deps): update a crate",
17266 "fix(daemon): apply a chosen action",
17267 ]);
17268 state.repo = repo.clone();
17269 state.base_branch = "main".to_owned();
17270 state.base_commit = c1;
17271 refresh_reviewed_commits(&mut state, "feat/own").await;
17272 assert_eq!(
17273 state.reviewed_commits,
17274 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17275 );
17276 assert_eq!(
17277 review_title(&state).as_deref(),
17278 Some("fix(daemon): apply a chosen action")
17279 );
17280 assert_eq!(
17281 leaked_subjects(&state, "feat/own").await,
17282 Some(vec!["chore(deps): update a crate".to_owned()])
17283 );
17284 state.config.merge.remote = "nowhere".to_owned();
17287 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17288 }
17289
17290 struct SummaryFixture {
17294 _tmp: tempfile::TempDir,
17295 repo: PathBuf,
17296 reply: PathBuf,
17297 runner: Runner,
17298 }
17299
17300 fn summary_fixture() -> SummaryFixture {
17301 ask_test_home();
17302 let tmp = tempfile::tempdir().unwrap();
17303 let repo = tmp.path().join("repo");
17304 std::fs::create_dir_all(&repo).unwrap();
17305 init_repo(&repo);
17306 let origin = tmp.path().join("origin.git");
17307 let g = |args: &[&str]| {
17308 let out = std::process::Command::new("git")
17309 .args(args)
17310 .current_dir(&repo)
17311 .quiet()
17312 .output()
17313 .expect("spawn git");
17314 assert!(
17315 out.status.success(),
17316 "git {args:?}: {}",
17317 String::from_utf8_lossy(&out.stderr)
17318 );
17319 };
17320 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17321 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17322 g(&["push", "-q", "origin", "main"]);
17323 g(&["checkout", "-q", "-b", "feat/own"]);
17324 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17325 g(&["add", "-A"]);
17326 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17327 let reply = tmp.path().join("reply.txt");
17328 std::fs::write(
17329 &reply,
17330 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17331 )
17332 .unwrap();
17333 let mut runner = runner_at(RunStatus::Gating);
17334 let mut state = review_state(&["feat(own): add the own file"]);
17335 state.repo = repo.clone();
17336 state.base_branch = "main".to_owned();
17337 state.candidates[0].branch = "feat/own".to_owned();
17338 state.candidates[0].worktree = repo.clone();
17339 let mut author = spec("alpha");
17340 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17341 author
17342 .env
17343 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17344 state.config.agents = vec![author];
17345 runner.state = state;
17346 SummaryFixture {
17347 _tmp: tmp,
17348 repo,
17349 reply,
17350 runner,
17351 }
17352 }
17353
17354 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17355 let winner = f.runner.state.candidates[0].clone();
17356 f.runner
17357 .guarded_pr_message(&winner, None, true)
17358 .await
17359 .unwrap()
17360 .expect("an English description never parks")
17361 }
17362
17363 fn clean_round_with_minor_findings() -> ReviewRound {
17364 let mut round = passing_round(2, None);
17365 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17366 round.reviews = vec![ReviewRecord {
17367 attempts: 0,
17368 reviewer: 1,
17369 agent: "alpha".into(),
17370 summary: "内部レビュー".into(),
17371 findings: vec![
17372 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17373 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17374 ],
17375 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17376 failed: None,
17377 duration_ms: 0,
17378 }];
17379 round
17380 }
17381
17382 #[tokio::test]
17383 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17384 for review_only in [false, true] {
17385 let mut f = summary_fixture();
17386 if !review_only {
17387 f.runner.state.candidates[0].agent = "alpha".into();
17388 }
17389 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17390 assert!(f.runner.state.reviews[0].clean);
17391 assert!(f.runner.state.reviews[0].fix.is_none());
17392 assert!(f.runner.state.open_findings().is_empty());
17393 assert_eq!(
17394 crate::pr_outcome::required_finding_ids(&f.runner.state),
17395 ["R2-1-1", "R2-1-2"]
17396 );
17397 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17398 let evidence = crate::pr_outcome::summary_evidence(
17399 &f.runner.state,
17400 crate::pr_outcome::FinalHead::At(&head),
17401 );
17402 assert!(
17403 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17404 );
17405 let omitted = posted(&mut f).await;
17407 assert!(omitted.body.contains("No description"));
17408 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17409 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17410 assert!(f.runner.state.change_summary.is_none());
17411 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17412 let described = posted(&mut f).await;
17413 assert!(described.body.contains("improve documentation for callers"));
17414 assert!(described.body.contains("remaining edge case"));
17415 assert!(described.body.contains("## Open review findings"));
17416 assert!(!described.body.contains("未修正"));
17417 }
17418 }
17419
17420 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17421 let mut f = summary_fixture();
17422 if !review_only {
17423 f.runner.state.candidates[0].agent = "alpha".into();
17424 }
17425 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17426 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17427 fixed
17428 .fix
17429 .as_mut()
17430 .unwrap()
17431 .rejected
17432 .push(crate::verdict::Rejection {
17433 id: "R1-1-1".into(),
17434 why: "呼び出されないコードのため辞退".into(),
17435 });
17436 let mut clean = clean_round_with_minor_findings();
17437 clean.verified_head = Some(head.clone());
17438 f.runner.state.reviews = vec![fixed, clean];
17439 ok_gate(&mut f.runner.state, Some(&head));
17440 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17441 f.runner.state.config.agents.push(scripted("beta", reply));
17442 f.runner.state.config.roles.language_judge =
17443 Some(crate::config::AgentChoice::One("beta".into()));
17444 f
17445 }
17446
17447 #[tokio::test]
17448 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17449 for review_only in [false, true] {
17450 let mut f = rewrite_fixture_with_findings(review_only,
17451 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17452 let m = posted(&mut f).await;
17453 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17454 let expected =
17455 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17456 assert!(m.body.contains(&expected));
17457 assert!(m.body.contains("## Review rounds"));
17458 assert!(
17459 m.body
17460 .contains("Gate (`verify.gate`): passed on the final commit")
17461 );
17462 assert!(
17463 m.body
17464 .contains("Tests (`verify.e2e`): passed on the final commit")
17465 );
17466 let c = f.runner.state.change_summary.as_ref().unwrap();
17467 assert!(
17468 m.body
17469 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17470 );
17471 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17472 assert!(crate::pr_outcome::covers_required_findings(
17473 &f.runner.state,
17474 &m.body
17475 ));
17476 assert!(m.body.contains("documentation remains incomplete"));
17477 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17478 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17479 assert!(f.runner.state.github_text.is_none());
17480 }
17481 }
17482
17483 #[tokio::test]
17484 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17485 for review_only in [false, true] {
17486 for reply in [
17487 "TITLE: fix: public description\nA short English description.",
17488 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17489 ] {
17490 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17491 let winner = f.runner.state.candidates[0].clone();
17492 let posted = f
17493 .runner
17494 .guarded_pr_message(&winner, None, true)
17495 .await
17496 .unwrap();
17497 assert!(posted.is_none());
17498 assert!(f.runner.state.parked);
17499 let withheld = f.runner.state.github_text.as_ref().unwrap();
17500 assert!(
17501 withheld.rewrite.is_none(),
17502 "an incomplete rewrite must not be selectable"
17503 );
17504 let q = ask::Questions::open()
17505 .get(withheld.question.as_ref().unwrap())
17506 .unwrap();
17507 assert!(
17508 !q.choices
17509 .contains(&crate::github_text::USE_REWRITE.to_owned())
17510 );
17511 assert!(
17512 q.detail.contains("R1-1-1")
17513 && q.detail.contains("R2-1-1")
17514 && q.detail.contains("R2-1-2")
17515 );
17516 }
17517 }
17518 }
17519
17520 #[tokio::test]
17521 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17522 for review_only in [false, true] {
17523 let mut f = summary_fixture();
17524 if !review_only {
17525 f.runner.state.candidates[0].agent = "alpha".into();
17526 }
17527 f.runner.state.instruction = "内部タスクの説明".into();
17528 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17529 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17530 f.runner.state.reviews = vec![
17531 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17532 round_with_notes(2, Some("二回目の内部修正報告です。")),
17533 passing_round(3, Some(&head)),
17534 ];
17535 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17536 ok_gate(&mut f.runner.state, Some(&head));
17537 let captured = f.repo.join("public-prompt.txt");
17538 let author = &mut f.runner.state.config.agents[0];
17539 author.command = vec![
17540 "sh".into(),
17541 "-c".into(),
17542 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17543 ];
17544 author.env.insert(
17545 "PROMPT_FILE".into(),
17546 captured.to_string_lossy().into_owned(),
17547 );
17548 let m = posted(&mut f).await;
17549 assert!(m.body.contains("Adds `own.txt`"));
17550 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17551 assert!(m.body.contains("R1-1-1"));
17552 assert!(
17553 m.body
17554 .contains("Gate (`verify.gate`): passed on the final commit")
17555 );
17556 assert!(
17557 m.body
17558 .contains("Tests (`verify.e2e`): passed on the final commit")
17559 );
17560 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17561 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17562 let prompt = std::fs::read_to_string(captured).unwrap();
17563 assert!(prompt.contains(&format!(
17564 "git diff {} {head}",
17565 f.runner.state.change_summary.as_ref().unwrap().base
17566 )));
17567 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17568 assert_eq!(
17569 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17570 "二回目の内部修正報告です。"
17571 );
17572 }
17573 }
17574
17575 #[tokio::test]
17576 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17577 for review_only in [false, true] {
17578 let mut f = summary_fixture();
17579 if !review_only {
17580 f.runner.state.candidates[0].agent = "alpha".into();
17581 }
17582 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17583 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17584 std::fs::write(&f.reply, "no usable description").unwrap();
17585 let failed = posted(&mut f).await;
17586 assert!(
17587 failed
17588 .body
17589 .contains("No description of what this change does could be generated")
17590 );
17591 assert!(failed.body.is_ascii());
17592 assert!(f.runner.state.github_text.is_none());
17593 assert!(f.runner.state.change_summary.is_none());
17594 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17595 let evidence = crate::pr_outcome::summary_evidence(
17596 &f.runner.state,
17597 crate::pr_outcome::FinalHead::At(&head),
17598 );
17599 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17600 assert!(
17601 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17602 .exists()
17603 );
17604 f.runner.state =
17605 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17606 std::fs::write(
17607 &f.reply,
17608 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17609 )
17610 .unwrap();
17611 let retried = posted(&mut f).await;
17612 assert!(retried.body.contains("Adds the committed own file."));
17613 assert_eq!(
17614 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17615 "日本語の修正報告"
17616 );
17617 }
17618 }
17619
17620 #[tokio::test]
17621 async fn same_head_verification_and_findings_invalidate_public_prose() {
17622 let mut f = summary_fixture();
17623 posted(&mut f).await;
17624 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17625 let original = f
17626 .runner
17627 .state
17628 .change_summary
17629 .as_ref()
17630 .unwrap()
17631 .evidence
17632 .clone();
17633 ok_gate(&mut f.runner.state, Some(&head));
17634 std::fs::write(
17635 &f.reply,
17636 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17637 )
17638 .unwrap();
17639 let verified = posted(&mut f).await;
17640 assert!(verified.body.contains("now verified"));
17641 assert_ne!(
17642 f.runner.state.change_summary.as_ref().unwrap().evidence,
17643 original
17644 );
17645 let mut round = round_with_notes(1, Some("内部報告"));
17646 round.reviews.push(ReviewRecord {
17647 attempts: 0,
17648 reviewer: 1,
17649 agent: "alpha".into(),
17650 summary: "内部レビュー".into(),
17651 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17652 vote: None,
17653 failed: None,
17654 duration_ms: 0,
17655 });
17656 round
17657 .fix
17658 .as_mut()
17659 .unwrap()
17660 .rejected
17661 .push(crate::verdict::Rejection {
17662 id: "R1-1-1".into(),
17663 why: "呼び出し元から到達できない".into(),
17664 });
17665 f.runner.state.reviews = vec![round];
17666 let omitted = posted(&mut f).await;
17668 assert!(omitted.body.contains("No description"));
17669 assert!(omitted.body.contains("R1-1-1"));
17670 assert!(!omitted.body.contains("到達"));
17671 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17672 let described = posted(&mut f).await;
17673 assert!(described.body.contains("unreachable from callers"));
17674 assert!(described.body.contains("R1-1-1"));
17675 assert!(!described.body.contains("到達"));
17676 let evidence = f
17677 .runner
17678 .state
17679 .change_summary
17680 .as_ref()
17681 .unwrap()
17682 .evidence
17683 .clone();
17684 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17685 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17686 assert!(posted(&mut f).await.body.contains("rationale updated"));
17687 assert_ne!(
17688 f.runner.state.change_summary.as_ref().unwrap().evidence,
17689 evidence
17690 );
17691 f.runner
17693 .state
17694 .change_summary
17695 .as_mut()
17696 .unwrap()
17697 .evidence
17698 .clear();
17699 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17700 assert!(posted(&mut f).await.body.contains("current specification"));
17701 }
17702
17703 #[tokio::test]
17704 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17705 let mut f = summary_fixture();
17706 let m = posted(&mut f).await;
17707 assert_eq!(m.title, "feat(own): add the own file");
17708 assert!(
17709 m.body
17710 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17711 "{}",
17712 m.body
17713 );
17714 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17715 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17716 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17717 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17718 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17719 let kept = f.runner.state.change_summary.as_ref().unwrap();
17720 assert_eq!(
17721 (kept.head.as_str(), kept.base.as_str()),
17722 (head.as_str(), base.as_str())
17723 );
17724 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17725 }
17726
17727 #[tokio::test]
17728 async fn review_only_summary_receives_the_configured_github_language() {
17729 let mut f = summary_fixture();
17730 f.runner.state.config.graph.github_language = "ja".into();
17731 let captured = f.repo.join("summary-prompt.txt");
17732 let author = &mut f.runner.state.config.agents[0];
17733 author.command = vec![
17734 "sh".into(),
17735 "-c".into(),
17736 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17737 ];
17738 author.env.insert(
17739 "PROMPT_FILE".into(),
17740 captured.to_string_lossy().into_owned(),
17741 );
17742 std::fs::write(
17743 &f.reply,
17744 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17745 )
17746 .unwrap();
17747 let m = posted(&mut f).await;
17748 let prompt = std::fs::read_to_string(captured).unwrap();
17749 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17750 assert!(!prompt.contains("in English"));
17751 assert_eq!(m.title, "feat(own): ファイルを追加");
17752 assert!(m.body.contains("own.txt を追加します。"));
17753 }
17754
17755 #[tokio::test]
17756 async fn review_fixes_come_after_the_whole_change_summary() {
17757 let mut f = summary_fixture();
17758 f.runner.state.reviews = vec![
17759 round_with_notes(1, Some("handled the PATH case")),
17760 round_with_notes(2, None),
17761 ];
17762 let m = posted(&mut f).await;
17763 let summary = m.body.find("## Summary").expect("summary");
17764 let fixes = m.body.find("## Review rounds").expect("review rounds");
17765 assert!(summary < fixes, "{}", m.body);
17766 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17767 assert!(m.body.contains("## Final verification"), "{}", m.body);
17768 }
17769
17770 #[tokio::test]
17771 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17772 let mut f = summary_fixture();
17773 let first = posted(&mut f).await;
17774 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17775 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17777 let again = posted(&mut f).await;
17778 assert_eq!((again.title, again.body), (first.title, first.body));
17779
17780 let g = |args: &[&str]| {
17781 let out = std::process::Command::new("git")
17782 .args(args)
17783 .current_dir(&f.repo)
17784 .quiet()
17785 .output()
17786 .expect("spawn git");
17787 assert!(out.status.success(), "git {args:?}");
17788 };
17789 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17790 g(&["add", "-A"]);
17791 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17792 let second = posted(&mut f).await;
17793 let kept = f.runner.state.change_summary.clone().unwrap();
17794 assert_ne!(kept.head, first_head);
17795 assert!(second.body.contains("Not read yet."), "{}", second.body);
17796 assert!(!second.body.contains("so the first summary is read"));
17797 assert_eq!(second.title, "feat(own): other");
17798 }
17799
17800 #[tokio::test]
17801 async fn an_unavailable_summary_is_stated_and_never_invented() {
17802 let mut f = summary_fixture();
17803 std::fs::write(&f.reply, "no title line here").unwrap();
17804 let m = posted(&mut f).await;
17805 assert!(
17806 m.body
17807 .contains("No description of what this change does could be generated"),
17808 "{}",
17809 m.body
17810 );
17811 assert!(m.body.contains("internal reports remain in local records"));
17812 assert!(f.runner.state.change_summary.is_none());
17813 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17814 let winner = f.runner.state.candidates[0].clone();
17816 let preview = f
17817 .runner
17818 .guarded_pr_message(&winner, None, false)
17819 .await
17820 .unwrap()
17821 .unwrap();
17822 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17823 }
17824
17825 #[test]
17826 fn pr_message_review_single_commit_uses_its_subject() {
17827 let state = review_state(&["feat(nats): per-role user"]);
17828 let m = pr_message(&state, 'A');
17829 assert_eq!(m.title, "feat(nats): per-role user");
17830 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17831 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17832 }
17833
17834 #[test]
17835 fn pr_message_review_multi_commit_takes_the_oldest() {
17836 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17837 let m = pr_message(&state, 'A');
17838 assert_eq!(m.title, "feat: the change");
17839 for s in ["feat: the change", "fix: typo", "fix: again"] {
17840 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17841 }
17842 }
17843
17844 #[test]
17845 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17846 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17847 let mut state = review_state(&[first, "fix: later fixup"]);
17848 state.candidates[0].branch = "機能/ログイン".to_owned();
17849 let m = pr_message(&state, 'A');
17850 assert!(
17851 m.title.starts_with("chore: land candidate A of run"),
17852 "{}",
17853 m.title
17854 );
17855 }
17856 }
17857
17858 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17859 BranchFacts {
17860 commits: commits
17861 .iter()
17862 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17863 .collect(),
17864 stat: stat.to_owned(),
17865 }
17866 }
17867
17868 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17869 let mut r = review_round(true, 0, 1, 1, true, true);
17870 r.round = round;
17871 r.fix = notes.map(|n| FixRecord {
17872 agent: "fixer".to_owned(),
17873 addressed: Vec::new(),
17874 rejected: Vec::new(),
17875 notes: n.to_owned(),
17876 committed: true,
17877 failed: None,
17878 duration_ms: 0,
17879 continuation: None,
17880 });
17881 r
17882 }
17883
17884 #[test]
17885 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17886 let state = review_state(&["ignored"]);
17887 let f = facts(
17888 &[
17889 (
17890 "fix(login): resolve PATH on macOS",
17891 "Login shells skip rc files.",
17892 ),
17893 ("fix: address review", ""),
17894 ],
17895 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17896 );
17897 let m = pr_message_with(&state, 'A', Some(&f));
17898 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17899 assert!(
17900 m.body
17901 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17902 "{}",
17903 m.body
17904 );
17905 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17906 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17907 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17908 for banned in [
17909 "Review the work already",
17910 "no task statement",
17911 "Original task",
17912 ] {
17913 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17914 }
17915 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17916 }
17917
17918 #[test]
17919 fn pr_message_review_truncates_a_huge_first_commit_body() {
17920 let state = review_state(&["ignored"]);
17921 let f = facts(
17922 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17923 "s",
17924 );
17925 let m = pr_message_with(&state, 'A', Some(&f));
17926 assert!(m.body.len() < 30_000, "{}", m.body.len());
17927 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17928 assert!(
17929 m.body.contains("- ... 1 more commit(s)"),
17930 "{}",
17931 m.body.len()
17932 );
17933 assert!(m.body.ends_with("magi:candidate-a\n"));
17934 }
17935
17936 #[test]
17937 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17938 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17939 assert_eq!(m.title, "feat: x");
17940 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17941 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17942 assert!(!m.body.contains("no task statement"), "{}", m.body);
17943 }
17944
17945 #[test]
17946 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17947 let mut state = review_state(&["日本語の件名"]);
17948 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17949 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17950 state.candidates[0].branch = "機能/ログイン".to_owned();
17951 assert!(
17952 pr_message(&state, 'A')
17953 .title
17954 .starts_with("chore: land candidate A")
17955 );
17956 }
17957
17958 #[test]
17959 fn pr_message_review_fixes_survive_a_clean_final_round() {
17960 let mut state = review_state(&["feat: x"]);
17961 state.reviews = vec![
17962 round_with_notes(1, Some("handled the PATH case")),
17963 round_with_notes(2, None),
17964 ];
17965 let body = pr_message(&state, 'A').body;
17966 assert!(
17967 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17968 "{body}"
17969 );
17970 assert!(body.contains("### Round 1"), "{body}");
17971
17972 state.reviews = vec![
17973 round_with_notes(1, Some("first fix")),
17974 round_with_notes(2, Some("")),
17975 round_with_notes(3, Some("second fix")),
17976 round_with_notes(4, None),
17977 ];
17978 let body = pr_message(&state, 'A').body;
17979 assert!(
17980 body.contains("### Round 1") && !body.contains("first fix"),
17981 "{body}"
17982 );
17983 assert!(
17984 body.contains("### Round 3") && !body.contains("second fix"),
17985 "{body}"
17986 );
17987 assert!(body.contains("### Round 2"), "{body}");
17988 }
17989
17990 const FINAL: &str = "abcdef1234567890";
17991
17992 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17993 let mut r = round_with_notes(round, None);
17994 r.verified_head = verified.map(str::to_owned);
17995 r
17996 }
17997
17998 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17999 state.gate = vec![CommandOutcome {
18000 command: "gate".to_owned(),
18001 code: Some(0),
18002 output_tail: String::new(),
18003 duration_ms: 0,
18004 resource_blocked: false,
18005 }];
18006 state.gate_ran = true;
18007 state.gate_head = head.map(str::to_owned);
18008 }
18009
18010 fn body_at(state: &RunState) -> String {
18011 pr_message_raw(
18012 state,
18013 'A',
18014 None,
18015 crate::pr_outcome::FinalHead::At(FINAL),
18016 ChangeNote::NotAsked,
18017 )
18018 .body
18019 }
18020
18021 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18022
18023 #[test]
18024 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18025 let mut state = review_state(&["feat: x"]);
18026 state.reviews = vec![
18027 round_with_notes(1, Some(SANDBOXED)),
18028 passing_round(2, Some(FINAL)),
18029 ];
18030 ok_gate(&mut state, Some(FINAL));
18031 let body = body_at(&state);
18032 assert!(!body.contains("Handled the PATH case."), "{body}");
18033 assert!(!body.contains("could not commit"), "{body}");
18034 assert!(!body.contains("could not run"), "{body}");
18035 assert!(
18036 body.contains("The fix is committed on the branch"),
18037 "{body}"
18038 );
18039 assert!(!body.contains("sandbox"), "{body}");
18040 assert!(
18041 body.contains("Tests (`verify.e2e`): passed on the final commit"),
18042 "{body}"
18043 );
18044 assert!(
18045 body.contains("Gate (`verify.gate`): passed on the final commit"),
18046 "{body}"
18047 );
18048 assert!(
18050 state.reviews[0]
18051 .fix
18052 .as_ref()
18053 .unwrap()
18054 .notes
18055 .contains("could not commit")
18056 );
18057 }
18058
18059 #[test]
18060 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18061 let mut state = review_state(&["feat: x"]);
18062 let mut r1 = round_with_notes(1, Some(SANDBOXED));
18063 r1.fix.as_mut().unwrap().committed = false;
18064 let mut r2 = passing_round(2, Some(FINAL));
18065 r2.e2e[0].code = Some(1);
18066 state.reviews = vec![r1, r2];
18067 let body = body_at(&state);
18068 assert!(!body.contains("could not commit"), "{body}");
18069 assert!(!body.contains("could not run"), "{body}");
18070 assert!(body.contains("This round produced no commit"), "{body}");
18071 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18072 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18073 assert!(!body.contains("passed"), "{body}");
18074 }
18075
18076 #[test]
18077 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18078 let mut state = review_state(&["feat: x"]);
18079 state.reviews = vec![
18080 round_with_notes(1, Some(SANDBOXED)),
18081 passing_round(2, Some("0123456789abcdef")),
18082 ];
18083 ok_gate(&mut state, Some("0123456789abcdef"));
18084 let body = body_at(&state);
18085 assert!(!body.contains("HTTP tests could not run"), "{body}");
18086 assert!(!body.contains("could not commit"), "{body}");
18088 assert!(
18089 body.contains("passed on an earlier commit (`0123456`)"),
18090 "{body}"
18091 );
18092 assert!(!body.contains("passed on the final commit"), "{body}");
18093
18094 ok_gate(&mut state, None);
18096 let body = body_at(&state);
18097 assert!(
18098 body.contains("passed on a commit that was not recorded"),
18099 "{body}"
18100 );
18101 }
18102
18103 #[test]
18104 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18105 let state = state_with_summary(
18106 "add retries to the client",
18107 "TITLE: feat: retries\n\nDid it.",
18108 );
18109 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18110 assert_eq!(m.title, "feat: retries");
18111 assert!(
18112 m.body.contains("<summary>Original task</summary>"),
18113 "{}",
18114 m.body
18115 );
18116 assert!(!m.body.contains("Commits under review"), "{}", m.body);
18117 assert!(
18118 m.body
18119 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18120 "{}",
18121 m.body
18122 );
18123 }
18124
18125 #[test]
18126 fn pr_message_review_bounds_a_long_english_subject() {
18127 let long = format!("feat: {}", "word ".repeat(100));
18128 let m = pr_message(&review_state(&[&long]), 'A');
18129 assert!(m.title.starts_with("feat: word"), "{}", m.title);
18130 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18131 }
18132
18133 #[test]
18134 fn pr_message_implementation_run_is_unchanged_by_review_support() {
18135 let state = state_with_summary("add retries\n\ndetails", "- did some things");
18136 let m = pr_message(&state, 'A');
18137 assert_eq!(m.title, "add retries");
18138 assert!(m.body.contains("<summary>Original task</summary>"));
18139 assert!(!m.body.contains("Commits under review"));
18140 assert_eq!(landing_subject_source(&state), state.instruction);
18141 }
18142
18143 #[test]
18144 fn review_run_squash_subject_is_the_change_not_the_prompt() {
18145 let state = review_state(&["feat: the change", "fix: typo"]);
18146 let source = landing_subject_source(&state);
18147 assert_eq!(land::merge_subject("", &source), "feat: the change");
18148 assert_eq!(
18149 land::merge_subject("magi: candidate A (uncommitted work)", &source),
18150 "feat: the change"
18151 );
18152 assert_eq!(
18154 land::merge_subject("feat: renamed by hand", &source),
18155 "feat: renamed by hand"
18156 );
18157 let blank = review_state(&["日本語"]);
18158 assert!(
18159 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18160 );
18161 }
18162
18163 #[test]
18164 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18165 let state = review_state(&["feat: the change"]);
18166 let source = landing_subject_source(&state);
18167 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18168 assert_eq!(
18169 land::merge_subject(landing_title(&state, old), &source),
18170 "feat: the change"
18171 );
18172 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18173 let task = state_with_summary("add retries", "");
18174 assert_eq!(landing_title(&task, old), old);
18175 }
18176
18177 #[test]
18178 fn pr_message_describes_the_change_not_the_task() {
18179 let state = state_with_summary(
18180 "今回やってほしいこと: results projector を直す",
18181 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18182 );
18183 let m = pr_message(&state, 'A');
18184 assert_eq!(m.title, "fix(web): batch the runs list reads");
18185 assert!(
18186 m.body.starts_with("## Summary\n\n- reads run.json once"),
18187 "{}",
18188 m.body
18189 );
18190 assert!(!m.body.contains("TITLE:"), "{}", m.body);
18191 let task_at = m.body.find("今回やってほしいこと").unwrap();
18192 let details_at = m.body.find("<details>").unwrap();
18193 assert!(
18194 details_at < task_at,
18195 "the task lives inside <details>: {}",
18196 m.body
18197 );
18198 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18199 assert!(m.body.contains("magi:candidate-a"));
18200 }
18201
18202 #[test]
18203 fn pr_message_falls_back_to_the_task_without_a_title_line() {
18204 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18205 let m = pr_message(&state, 'A');
18206 assert_eq!(m.title, "add retries");
18207 assert!(
18208 m.body.contains("## Summary\n\n- did some things"),
18209 "{}",
18210 m.body
18211 );
18212
18213 let none = RunState::new(
18214 PathBuf::from("/repo"),
18215 "main".to_owned(),
18216 "abc1234".to_owned(),
18217 "add retries".to_owned(),
18218 Config::default(),
18219 );
18220 let m = pr_message(&none, 'A');
18221 assert_eq!(m.title, "add retries");
18222 assert!(!m.body.contains("## Summary"), "{}", m.body);
18223 }
18224
18225 #[test]
18226 fn pr_message_refuses_the_candidate_commit_subject() {
18227 for bad in [
18228 "TITLE: magi: candidate A (uncommitted work)",
18229 "TITLE: chore: stuff (uncommitted work)",
18230 "TITLE: ",
18231 ] {
18232 let state = state_with_summary("add retries", bad);
18233 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18234 }
18235 }
18236
18237 #[test]
18238 fn pr_message_bounds_a_very_long_task_and_title() {
18239 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18240 let state = state_with_summary(&long, "- nothing");
18241 let m = pr_message(&state, 'A');
18242 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18243 assert!(!m.title.contains('\n'));
18244
18245 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18246 let m = pr_message(&state, 'A');
18247 assert!(m.title.starts_with("feat: "));
18248 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18249 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18250 }
18251
18252 fn long_title_of(instruction: &str) -> String {
18253 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18254 }
18255
18256 #[test]
18257 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18258 let first = "Make the landing path keep a readable title for long tasks";
18259 let line = format!(
18260 "{first}. {}",
18261 "And then keep going with more words ".repeat(20)
18262 );
18263 let t = long_title_of(&line);
18264 assert_eq!(t, first);
18265 assert!(!t.starts_with("chore: land"));
18266 }
18267
18268 #[test]
18269 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18270 let line = "word ".repeat(200);
18271 let t = long_title_of(&line);
18272 assert!(t.ends_with("word..."), "{t}");
18273 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18274 }
18275
18276 #[test]
18277 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18278 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18279 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18280 }
18281 }
18282
18283 #[test]
18284 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18285 let mut state = state_with_summary("再試行を追加する", "- nothing");
18286 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18287 state.config.graph.github_language = "ja".into();
18288 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18289 }
18290
18291 #[test]
18292 fn pr_message_title_limit_is_exact() {
18293 let at = "a".repeat(PR_TITLE_MAX);
18294 assert_eq!(long_title_of(&at), at);
18295 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18296 assert!(over.ends_with("..."), "{over}");
18297 assert_eq!(over.chars().count(), PR_TITLE_MAX);
18298 }
18299
18300 #[test]
18301 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18302 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18303 let t = long_title_of(&line);
18304 assert!(t.ends_with("..."), "{t}");
18305 assert!(t.is_ascii(), "{t}");
18306 }
18307
18308 #[test]
18309 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18310 let line = format!(
18311 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18312 "plus more filler words ".repeat(20)
18313 );
18314 let t = long_title_of(&line);
18315 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18316 }
18317
18318 #[test]
18319 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18320 let line = format!("\n\n# {}", "title words ".repeat(40));
18321 let state = state_with_summary(&line, "- nothing");
18322 let m = pr_message(&state, 'A');
18323 assert!(m.title.starts_with("title words"), "{}", m.title);
18324 assert_eq!(
18325 land::merge_subject(&m.title, &landing_subject_source(&state)),
18326 m.title
18327 );
18328 assert_eq!(
18330 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18331 "feat: renamed by hand"
18332 );
18333 }
18334
18335 #[tokio::test]
18336 async fn github_text_rewrite_is_bounded_and_falls_back() {
18337 crate::run::pin_test_home();
18338 for (reply, accepted) in [
18339 (
18340 "TITLE: fix: retries\nAdd retries for failed requests.",
18341 true,
18342 ),
18343 (
18344 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18345 false,
18346 ),
18347 (
18348 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18349 true,
18350 ),
18351 (
18352 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18353 true,
18354 ),
18355 (
18356 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18357 true,
18358 ),
18359 (
18360 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18361 false,
18362 ),
18363 ("TITLE: fix: retries\n", false),
18364 ] {
18365 let dir = tempfile::tempdir().unwrap();
18366 let mut runner = runner_at(RunStatus::Gating);
18367 runner.state = state_with_summary(
18368 "add retries",
18369 "TITLE: fix: retries\n日本語の説明を書きます。",
18370 );
18371 runner.state.repo = dir.path().to_owned();
18372 runner.state.candidates[0].worktree = dir.path().to_owned();
18373 let mut author = spec("alpha");
18374 author.command = vec![
18375 "sh".into(),
18376 "-c".into(),
18377 "printf '%s' \"$REWRITE_REPLY\"".into(),
18378 ];
18379 author.env.insert("REWRITE_REPLY".into(), reply.into());
18380 runner.state.config.agents = vec![author];
18381 seed_public_summary(&mut runner);
18382 let winner = runner.state.candidates[0].clone();
18383 let first = runner
18384 .guarded_pr_message(&winner, None, true)
18385 .await
18386 .unwrap();
18387 let message = if accepted {
18388 first.expect("a passing rewrite never asks")
18389 } else {
18390 assert!(first.is_none());
18392 assert!(runner.state.parked);
18393 let store = ask::Questions::open();
18394 let asked: Vec<_> = store
18395 .list()
18396 .into_iter()
18397 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18398 .collect();
18399 assert_eq!(asked.len(), 1);
18400 assert!(!asked[0].detail.contains("secret"));
18401 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18402 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18403 assert_eq!(
18404 asked[0]
18405 .choices
18406 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18407 rewrite.is_some()
18408 );
18409 store
18412 .update(&asked[0].id, |q| {
18413 q.abandon("test");
18414 Ok(())
18415 })
18416 .unwrap();
18417 let again = runner
18418 .guarded_pr_message(&winner, None, true)
18419 .await
18420 .unwrap()
18421 .expect("an abandoned question falls back");
18422 let n = store
18423 .list()
18424 .into_iter()
18425 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18426 .count();
18427 assert_eq!(n, 1, "asked once per rejected text");
18428 again
18429 };
18430 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18431 assert!(!message.body.contains("secret"));
18432 assert!(!message.body.contains("prefix"));
18433 assert!(!message.title.contains("secret"));
18434 assert_eq!(
18435 message.body.contains("Add retries for failed requests."),
18436 accepted
18437 );
18438 assert_eq!(
18439 runner.state.seats["github-text-rewrite"].turns, 1,
18440 "only one rewrite invocation"
18441 );
18442 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18443 if accepted {
18444 assert!(runner.state.github_text.is_none());
18445 assert!(!runner.state.parked);
18446 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18447 assert!(message.body.contains("## Final verification"));
18448 assert!(message.body.contains("Evidence: final diff"));
18449 assert!(
18450 message
18451 .body
18452 .contains(&format!("magi:run/{}", runner.state.id))
18453 );
18454 }
18455 if !accepted {
18456 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18457 assert!(
18458 message
18459 .body
18460 .contains(&format!("magi:run/{}", runner.state.id))
18461 );
18462 }
18463 }
18464 }
18465
18466 fn scripted(id: &str, reply: &str) -> AgentSpec {
18467 let mut a = spec(id);
18468 a.command = vec![
18469 "sh".into(),
18470 "-c".into(),
18471 "printf '%s' \"$REWRITE_REPLY\"".into(),
18472 ];
18473 a.env.insert("REWRITE_REPLY".into(), reply.into());
18474 a
18475 }
18476
18477 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18478 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18479
18480 fn seed_public_summary(runner: &mut Runner) {
18482 let repo = &runner.state.repo;
18483 let g = |args: &[&str]| {
18484 let out = std::process::Command::new("git")
18485 .args(args)
18486 .current_dir(repo)
18487 .quiet()
18488 .output()
18489 .unwrap();
18490 assert!(out.status.success(), "git {args:?}");
18491 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18492 };
18493 g(&["init", "-q", "-b", "main"]);
18494 g(&[
18495 "-c",
18496 "user.name=Test",
18497 "-c",
18498 "user.email=test@example.invalid",
18499 "commit",
18500 "-q",
18501 "--allow-empty",
18502 "-m",
18503 "feat: fixture",
18504 ]);
18505 let head = g(&["rev-parse", "HEAD"]);
18506 g(&["update-ref", "refs/remotes/origin/main", &head]);
18507 let evidence = crate::pr_outcome::summary_evidence(
18508 &runner.state,
18509 crate::pr_outcome::FinalHead::At(&head),
18510 );
18511 let generated = &runner.state.candidates[0].summary;
18512 runner.state.change_summary = Some(crate::run::ChangeSummary {
18513 base: head.clone(),
18514 head,
18515 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18516 title: summary_title(generated).unwrap(),
18517 body: summary_without_title(generated),
18518 });
18519 }
18520
18521 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18522 let mut runner = runner_at(RunStatus::Gating);
18523 runner.state = state_with_summary("add retries", JA_SUMMARY);
18524 if review_only {
18525 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18526 }
18527 runner.state.repo = dir.to_owned();
18528 runner.state.candidates[0].worktree = dir.to_owned();
18529 runner.state.config.agents = agents;
18530 seed_public_summary(&mut runner);
18531 runner
18532 }
18533
18534 fn gate_questions(runner: &Runner) -> usize {
18535 ask::Questions::open()
18536 .list()
18537 .into_iter()
18538 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18539 .count()
18540 }
18541
18542 #[tokio::test]
18543 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18544 crate::run::pin_test_home();
18545 let dir = tempfile::tempdir().unwrap();
18546 let mut runner = rewrite_runner(
18547 dir.path(),
18548 true,
18549 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18550 );
18551 runner.state.config.roles.language_judge =
18552 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18553 let winner = runner.state.candidates[0].clone();
18554 let msg = runner
18555 .guarded_pr_message(&winner, None, true)
18556 .await
18557 .unwrap()
18558 .expect("an English rewrite posts without asking");
18559 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18560 assert!(msg.body.contains("<details>kept</details>"));
18561 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18562 assert_eq!(gate_questions(&runner), 0);
18563 assert!(runner.state.github_text.is_none());
18564 assert!(!runner.state.seats.contains_key("impl-A"));
18565 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18566 }
18567
18568 #[tokio::test]
18569 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18570 crate::run::pin_test_home();
18571 let dir = tempfile::tempdir().unwrap();
18572 let mut runner = rewrite_runner(
18573 dir.path(),
18574 false,
18575 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18576 );
18577 runner.state.config.roles.language_judge =
18578 Some(crate::config::AgentChoice::One("beta".into()));
18579 runner.state.seats.insert(
18580 "impl-A".into(),
18581 SeatState::new("impl-A", "alpha", runner.state.seed),
18582 );
18583 let winner = runner.state.candidates[0].clone();
18584 let msg = runner
18585 .guarded_pr_message(&winner, None, true)
18586 .await
18587 .unwrap()
18588 .expect("rewritten");
18589 assert!(msg.body.contains("<details>kept</details>"));
18590 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18591 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18592 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18593 }
18594
18595 #[tokio::test]
18596 async fn rewrite_reuses_a_resumable_implementer_session() {
18597 crate::run::pin_test_home();
18598 let dir = tempfile::tempdir().unwrap();
18599 let mut runner = rewrite_runner(
18600 dir.path(),
18601 false,
18602 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18603 );
18604 runner.state.config.roles.language_judge =
18605 Some(crate::config::AgentChoice::One("beta".into()));
18606 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18607 seat.turns = 1;
18608 runner.state.seats.insert("impl-A".into(), seat);
18609 let winner = runner.state.candidates[0].clone();
18610 runner
18611 .guarded_pr_message(&winner, None, true)
18612 .await
18613 .unwrap()
18614 .expect("rewritten");
18615 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18616 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18617 }
18618
18619 #[tokio::test]
18620 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
18621 crate::run::pin_test_home();
18624 let dir = tempfile::tempdir().unwrap();
18625 let mut exhausted = spec("alpha");
18626 exhausted.command = vec!["false".into()];
18627 let mut runner = rewrite_runner(
18628 dir.path(),
18629 false,
18630 vec![exhausted, scripted("beta", EN_REWRITE)],
18631 );
18632 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18634 "alpha".into(),
18635 "beta".into(),
18636 ]));
18637 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18638 seat.turns = 1;
18639 runner.state.seats.insert("impl-A".into(), seat);
18640 let winner = runner.state.candidates[0].clone();
18641 let msg = runner
18642 .guarded_pr_message(&winner, None, true)
18643 .await
18644 .unwrap()
18645 .expect("the role rewrites after the implementer fails");
18646 assert!(msg.body.contains("`src/net.rs`"));
18647 assert_eq!(
18648 runner.state.seats["impl-A"].turns, 2,
18649 "the implementer was tried once"
18650 );
18651 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18652 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18653 }
18654
18655 #[tokio::test]
18656 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18657 crate::run::pin_test_home();
18658 let dir = tempfile::tempdir().unwrap();
18659 let mut broken = spec("bad");
18660 broken.command = vec!["false".into()];
18661 let mut runner = rewrite_runner(
18662 dir.path(),
18663 true,
18664 vec![
18665 broken,
18666 scripted("noformat", "Add retries."),
18667 scripted("good", EN_REWRITE),
18668 ],
18669 );
18670 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18671 "bad".into(),
18672 "noformat".into(),
18673 "good".into(),
18674 ]));
18675 let winner = runner.state.candidates[0].clone();
18676 let msg = runner
18677 .guarded_pr_message(&winner, None, true)
18678 .await
18679 .unwrap()
18680 .expect("the third agent rewrites");
18681 assert!(msg.body.contains("`src/net.rs`"));
18682 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18683 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18684 }
18685
18686 #[tokio::test]
18687 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18688 crate::run::pin_test_home();
18689 for (agents, role, saved) in [
18691 (
18692 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18693 Some("beta"),
18694 true,
18695 ),
18696 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18697 (Vec::new(), None, false),
18698 ] {
18699 let dir = tempfile::tempdir().unwrap();
18700 let mut runner = rewrite_runner(dir.path(), true, agents);
18701 runner.state.config.roles.language_judge =
18702 role.map(|r| crate::config::AgentChoice::One(r.into()));
18703 let winner = runner.state.candidates[0].clone();
18704 let first = runner
18705 .guarded_pr_message(&winner, None, true)
18706 .await
18707 .unwrap();
18708 assert!(first.is_none(), "withheld text parks the run");
18709 assert!(runner.state.parked);
18710 assert_eq!(gate_questions(&runner), 1);
18711 let g = runner.state.github_text.as_ref().unwrap();
18712 assert_eq!(g.rewrite.is_some(), saved);
18713 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18714 }
18715 }
18716
18717 #[tokio::test]
18718 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18719 crate::run::pin_test_home();
18720 let dir = tempfile::tempdir().unwrap();
18721 let mut runner = runner_at(RunStatus::Gating);
18722 runner.state =
18723 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18724 runner.state.repo = dir.path().to_owned();
18725 runner.state.candidates[0].worktree = dir.path().to_owned();
18726 let mut author = spec("alpha");
18727 author.command = vec![
18728 "sh".into(),
18729 "-c".into(),
18730 "printf '%s' \"$REWRITE_REPLY\"".into(),
18731 ];
18732 author.env.insert(
18733 "REWRITE_REPLY".into(),
18734 format!(
18735 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18736 "English sentence. ".repeat(600)
18737 ),
18738 );
18739 runner.state.config.agents = vec![author];
18740 seed_public_summary(&mut runner);
18741 let winner = runner.state.candidates[0].clone();
18742 assert!(
18743 runner
18744 .guarded_pr_message(&winner, None, true)
18745 .await
18746 .unwrap()
18747 .is_none()
18748 );
18749 let record = runner.state.github_text.clone().unwrap();
18750 let rewrite = record.rewrite.clone().unwrap();
18751 assert_eq!(rewrite.failed, ["body-language"]);
18752 assert!(
18753 rewrite
18754 .body
18755 .ends_with(&format!("magi:run/{}", runner.state.id))
18756 );
18757 let store = ask::Questions::open();
18758 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18759 assert!(
18760 q.choices
18761 .contains(&crate::github_text::USE_REWRITE.to_owned())
18762 );
18763 for label in [
18764 "ORIGINAL",
18765 "FALLBACK",
18766 "REWRITE",
18767 "Rules still failed: body-language",
18768 "truncated",
18769 ] {
18770 assert!(q.detail.contains(label), "{label}");
18771 }
18772 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18773 &runner.state,
18774 &crate::github_text::artifact_name(&record.fingerprint),
18775 ))
18776 .unwrap();
18777 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18778 runner.state =
18780 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18781 store
18782 .update(&q.id, |q| {
18783 q.say("まだ日本語")?;
18784 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18785 })
18786 .unwrap();
18787 assert!(
18788 runner
18789 .guarded_pr_message(&winner, None, true)
18790 .await
18791 .unwrap()
18792 .is_none()
18793 );
18794 let retry = runner.state.github_text.as_ref().unwrap();
18795 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18796 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18797 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18798 assert!(
18799 q.posting_comparison
18800 .as_ref()
18801 .unwrap()
18802 .matches_detail(&q.detail)
18803 );
18804 store
18805 .update(&q.id, |q| {
18806 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18807 })
18808 .unwrap();
18809 let message = runner
18810 .guarded_pr_message(&winner, None, true)
18811 .await
18812 .unwrap()
18813 .unwrap();
18814 assert_eq!(message.title, rewrite.title);
18815 assert_eq!(message.body, rewrite.body);
18816 assert_eq!(
18817 runner.state.github_text.as_ref().unwrap().choice,
18818 Some(GithubTextChoice::Rewrite)
18819 );
18820 let again = runner
18821 .guarded_pr_message(&winner, None, true)
18822 .await
18823 .unwrap()
18824 .unwrap();
18825 assert_eq!(again.title, message.title);
18826 assert_eq!(again.body, message.body);
18827 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18828 assert!(
18829 !runner
18830 .state
18831 .events
18832 .iter()
18833 .any(|e| e.message.contains(&rewrite.title)
18834 || e.message.contains("English sentence."))
18835 );
18836 }
18837
18838 #[tokio::test]
18839 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18840 crate::run::pin_test_home();
18841 let dir = tempfile::tempdir().unwrap();
18842 let mut runner = runner_at(RunStatus::Gating);
18843 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18844 runner.state.repo = dir.path().to_owned();
18845 runner.state.candidates[0].worktree = dir.path().to_owned();
18846 seed_public_summary(&mut runner);
18847 let winner = runner.state.candidates[0].clone();
18848 assert!(
18849 runner
18850 .guarded_pr_message(&winner, None, true)
18851 .await
18852 .unwrap()
18853 .is_none()
18854 );
18855 let record = runner.state.github_text.as_ref().unwrap();
18856 assert!(record.rewrite.is_none());
18857 let mut old = serde_json::to_value(record).unwrap();
18859 old.as_object_mut().unwrap().remove("rewrite");
18860 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18861 assert!(old.rewrite.is_none());
18862 let id = record.question.clone().unwrap();
18863 ask::Questions::open()
18864 .update(&id, |q| {
18865 q.choices.push(crate::github_text::USE_REWRITE.into());
18866 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18867 })
18868 .unwrap();
18869 let message = runner
18870 .guarded_pr_message(&winner, None, true)
18871 .await
18872 .unwrap()
18873 .unwrap();
18874 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18875 assert_eq!(
18876 runner.state.github_text.as_ref().unwrap().choice,
18877 Some(GithubTextChoice::Fallback)
18878 );
18879 }
18880
18881 #[tokio::test]
18882 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18883 crate::run::pin_test_home();
18884 let dir = tempfile::tempdir().unwrap();
18885 let mut runner = runner_at(RunStatus::Gating);
18886 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18887 runner.state.repo = dir.path().to_owned();
18888 runner.state.candidates[0].worktree = dir.path().to_owned();
18889 seed_public_summary(&mut runner);
18890 let winner = runner.state.candidates[0].clone();
18891 assert!(
18892 runner
18893 .guarded_pr_message(&winner, None, true)
18894 .await
18895 .unwrap()
18896 .is_none()
18897 );
18898 let store = ask::Questions::open();
18899 let find = |run: &str| {
18900 store
18901 .list()
18902 .into_iter()
18903 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18904 .unwrap()
18905 };
18906 let run = runner.state.id.clone();
18907 let q = find(&run);
18908 let fp = runner
18909 .state
18910 .github_text
18911 .as_ref()
18912 .unwrap()
18913 .fingerprint
18914 .clone();
18915 let name = crate::github_text::artifact_name(&fp);
18916 let path = crate::run::artifact_path(&runner.state, &name);
18917 assert!(q.detail.contains(&path.display().to_string()));
18918 assert!(q.detail.contains("title-language: title"));
18919 let saved = std::fs::read_to_string(&path).expect("artifact written");
18920 assert!(saved.contains("日本語のタイトル"));
18921 assert!(q.cwd.is_none());
18922 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18923 let say = |id: &str, text: &str, choice: &str| {
18925 store
18926 .update(id, |q| {
18927 q.thread.push(ask::Turn {
18928 who: ask::Who::Operator,
18929 body: text.to_owned(),
18930 at: jiff::Timestamp::now(),
18931 note: None,
18932 });
18933 q.answer(ask::Answer::Choice(choice.to_owned()))
18934 })
18935 .unwrap();
18936 };
18937 say(&q.id, "まだ日本語", "use my text");
18938 assert!(
18939 runner
18940 .guarded_pr_message(&winner, None, true)
18941 .await
18942 .unwrap()
18943 .is_none()
18944 );
18945 let second = find(&run);
18946 assert_ne!(second.id, q.id);
18947 assert!(second.detail.contains("did not pass"));
18948 assert!(second.detail.contains("日本語のタイトル"));
18951 assert!(!second.detail.contains("まだ日本語"));
18952 assert!(second.detail.contains(&path.display().to_string()));
18953 say(&second.id, "fix: retry failed requests", "use my text");
18954 let message = runner
18955 .guarded_pr_message(&winner, None, true)
18956 .await
18957 .unwrap()
18958 .expect("a vetted replacement is posted");
18959 assert_eq!(message.title, "fix: retry failed requests");
18960 assert_eq!(
18961 runner
18962 .state
18963 .github_text
18964 .as_ref()
18965 .unwrap()
18966 .chosen_title
18967 .as_deref(),
18968 Some("fix: retry failed requests")
18969 );
18970 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18971 let again = runner
18973 .guarded_pr_message(&winner, None, true)
18974 .await
18975 .unwrap()
18976 .unwrap();
18977 assert_eq!(again.title, message.title);
18978 assert_eq!(
18979 store
18980 .list()
18981 .into_iter()
18982 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18983 .count(),
18984 2
18985 );
18986 }
18987
18988 #[test]
18989 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18990 let mut state = state_with_summary(
18994 "add retries",
18995 "TITLE: fix(web): batch reads\n- reads run.json once",
18996 );
18997 state.config.graph.language = "ja".to_owned();
18998 let m = pr_message(&state, 'A');
18999 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19000 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19001
19002 let task = "今回やってほしいこと: results projector を直す";
19005 let mut state = state_with_summary(task, "- no title line");
19006 state.config.graph.language = "ja".to_owned();
19007 let m = pr_message(&state, 'A');
19008 assert_eq!(
19009 m.title,
19010 format!("chore: land candidate A of run {}", state.id)
19011 );
19012 assert!(
19013 m.body.contains(&format!(
19014 "<summary>Original task</summary>\n\n{task}\n\n</details>"
19015 )),
19016 "{}",
19017 m.body
19018 );
19019 }
19020
19021 #[test]
19022 fn pr_message_scrubs_home_paths_and_addresses() {
19023 let state = state_with_summary(
19024 "fix it in /Users/someone/src/x",
19025 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19026 );
19027 let m = pr_message(&state, 'A');
19028 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19029 assert!(!m.body.contains(leak), "{}", m.body);
19030 }
19031 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19032 }
19033
19034 #[test]
19035 fn pr_message_survives_a_task_that_closes_details() {
19036 let state = state_with_summary("a </details> b", "TITLE: fix: x");
19037 let m = pr_message(&state, 'A');
19038 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19039 }
19040
19041 #[test]
19042 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19043 let cmd = manual_merge_command(
19044 MergeStyle::Squash,
19045 Path::new("/repo"),
19046 "b",
19047 "fix: \"quoted\" $(x) `y`\n\nbody",
19048 );
19049 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19050 }
19051
19052 #[test]
19053 fn manual_merge_command_matches_the_configured_style() {
19054 let repo = Path::new("/repo");
19055 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19056
19057 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19058 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19059
19060 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19061 assert_eq!(
19062 squash,
19063 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19064 \"Merge magi run 0832 (candidate A)\""
19065 );
19066
19067 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19068 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19069 }
19070
19071 #[test]
19072 fn a_nudge_gets_a_quarter_of_the_budget() {
19073 assert_eq!(retry_budget(secs(1200), true), secs(300));
19075 assert_eq!(retry_budget(secs(3600), true), secs(900));
19076 }
19077
19078 #[test]
19079 fn a_resent_prompt_keeps_the_whole_budget() {
19080 assert_eq!(retry_budget(secs(1200), false), secs(1200));
19083 assert_eq!(retry_budget(secs(60), false), secs(60));
19084 }
19085
19086 #[test]
19087 fn the_floor_never_exceeds_the_original_budget() {
19088 assert_eq!(retry_budget(secs(60), true), secs(60));
19092 assert_eq!(retry_budget(secs(480), true), secs(120));
19093 assert_eq!(retry_budget(secs(0), true), secs(0));
19094 }
19095
19096 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19097 agent::CommandEvidence {
19098 id: "item1".to_owned(),
19099 description: "cargo test".to_owned(),
19100 exit_code,
19101 result_summary: String::new(),
19102 source: "codex".to_owned(),
19103 }
19104 }
19105
19106 #[test]
19107 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19108 assert!(!has_unconfirmed_command(&[]));
19112 }
19113
19114 #[test]
19115 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19116 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19120 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19121 assert!(!has_unconfirmed_command(&[
19122 evidence(Some(0)),
19123 evidence(Some(101))
19124 ]));
19125 }
19126
19127 #[test]
19128 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19129 assert!(has_unconfirmed_command(&[
19130 evidence(Some(0)),
19131 evidence(None)
19132 ]));
19133 }
19134
19135 #[test]
19136 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19137 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19138 assert_eq!(
19139 verified_noop_claim(true, &[], text).as_deref(),
19140 Some("already fixed by b32cfc4, on main.")
19141 );
19142 }
19143
19144 #[test]
19145 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19146 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19149 assert!(verified_noop_claim(false, &[], text).is_none());
19150 }
19151
19152 #[test]
19153 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19154 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19155 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19156 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19158 }
19159
19160 #[test]
19161 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19162 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19163 }
19164
19165 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19168 runner.state.candidates = shape
19169 .iter()
19170 .enumerate()
19171 .map(|(i, &(empty, verified))| Candidate {
19172 index: i,
19173 label: (b'A' + i as u8) as char,
19174 agent: "sonnet".to_owned(),
19175 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19176 worktree: PathBuf::from(format!("/wt/{i}")),
19177 summary: String::new(),
19178 stat: String::new(),
19179 files: 0,
19180 commits: 0,
19181 empty,
19182 failed: None,
19183 verified_noop: verified.map(str::to_owned),
19184 duration_ms: 0,
19185 folded: false,
19186 })
19187 .collect();
19188 }
19189
19190 #[test]
19191 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19192 ask_test_home();
19193 let mut runner = runner_at(RunStatus::Implementing);
19194 set_candidates(
19195 &mut runner,
19196 &[
19197 (true, Some("already on main at b32cfc4")),
19198 (true, Some("same fix, see the existing test")),
19199 ],
19200 );
19201
19202 runner
19203 .after_implement()
19204 .expect("a verified no-op is not an error");
19205
19206 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19207 }
19208
19209 #[test]
19210 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19211 ask_test_home();
19212 let mut runner = runner_at(RunStatus::Implementing);
19213 set_candidates(
19217 &mut runner,
19218 &[(true, Some("already on main at b32cfc4")), (true, None)],
19219 );
19220
19221 let err = runner
19222 .after_implement()
19223 .expect_err("an unverified empty candidate must still fail the run");
19224
19225 assert!(
19226 err.to_string().contains("no candidate produced a change"),
19227 "{err}"
19228 );
19229 assert_eq!(runner.state.status, RunStatus::Failed);
19230 }
19231
19232 #[test]
19233 fn after_implement_still_fails_an_ordinary_all_empty_run() {
19234 ask_test_home();
19235 let mut runner = runner_at(RunStatus::Implementing);
19236 set_candidates(&mut runner, &[(true, None), (true, None)]);
19237
19238 let err = runner
19239 .after_implement()
19240 .expect_err("no candidate declared anything; this is an ordinary failure");
19241
19242 assert!(
19243 err.to_string().contains("no candidate produced a change"),
19244 "{err}"
19245 );
19246 assert_eq!(runner.state.status, RunStatus::Failed);
19247 }
19248}
19249
19250#[derive(Debug)]
19253struct GraftRestoreFailed(String);
19254
19255impl std::fmt::Display for GraftRestoreFailed {
19256 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19257 write!(
19258 f,
19259 "could not restore the winner to its pre-graft commit: {}",
19260 self.0
19261 )
19262 }
19263}
19264
19265impl std::error::Error for GraftRestoreFailed {}
19266
19267#[cfg(test)]
19271mod resource_wait_tests {
19272 use super::*;
19273 use std::collections::BTreeMap;
19274
19275 fn sh(id: &str, script: &str) -> AgentSpec {
19276 AgentSpec {
19277 id: id.to_owned(),
19278 kind: crate::config::AgentKind::Command,
19279 model: None,
19280 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19281 extra_args: Vec::new(),
19282 env: BTreeMap::new(),
19283 prompt_delivery: None,
19284 max_turn_secs: None,
19285 }
19286 }
19287
19288 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19289 SeatJob {
19290 seat: SeatState::new("impl-A", &spec.id, 1),
19291 spec,
19292 cwd: dir.to_path_buf(),
19293 prompt: "do it".to_owned(),
19294 timeout,
19295 allow_write: true,
19296 sessions: false,
19297 artifacts: dir.join("art"),
19298 stem: "impl-A".to_owned(),
19299 handover: None,
19300 }
19301 }
19302
19303 fn state(resource_wait_secs: u64) -> RunState {
19304 let mut config = Config::default();
19305 config.graph.timeout_resource_wait = resource_wait_secs;
19306 RunState::new(
19307 PathBuf::from("/repo"),
19308 "main".to_owned(),
19309 "abc1234".to_owned(),
19310 "task".to_owned(),
19311 config,
19312 )
19313 }
19314
19315 fn hold(cache: &Path) -> crate::cache::Guard {
19316 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19317 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19318 crate::cache::AcquireOutcome::Acquired(g) => g,
19319 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19320 }
19321 }
19322
19323 async fn run(
19324 jobs: Vec<SeatJob>,
19325 sem: Arc<Semaphore>,
19326 cache: Option<&Path>,
19327 st: &mut RunState,
19328 ) -> Vec<(usize, SeatState, AgentOutcome)> {
19329 let prompts = Prompts::default();
19330 let ctx = WaveCtx {
19331 run: "r",
19332 node: "implement",
19333 prompts: &prompts,
19334 cache,
19335 round: None,
19336 carry_seats: false,
19337 pauses: None,
19338 };
19339 wave(jobs, sem, &ctx, st, 0).await
19340 }
19341
19342 #[tokio::test]
19343 async fn verification_cache_wait_preserves_the_command_budget() {
19344 crate::run::pin_test_home();
19345 let tmp = tempfile::tempdir().unwrap();
19346 let cache = tmp.path().join("target");
19347 let guard = hold(&cache);
19348 let release = tokio::spawn(async move {
19349 tokio::time::sleep(Duration::from_millis(100)).await;
19350 guard.release();
19351 });
19352 let mut st = state(10);
19353 let budget = Duration::from_millis(50);
19354 let wait = repository_wait(&st, None);
19355 let (outcomes, _) = with_cache_lease(
19356 &mut st,
19357 Some(&cache),
19358 "gate",
19359 "gate",
19360 tmp.path(),
19361 "abc1234",
19362 budget,
19363 "test gate",
19364 &wait,
19365 |_, available| async move {
19366 assert_eq!(
19367 available, budget,
19368 "contention must not shorten verification"
19369 );
19370 (Vec::new(), false, Vec::new())
19371 },
19372 )
19373 .await
19374 .unwrap();
19375 release.await.unwrap();
19376 assert!(outcomes.is_empty(), "{outcomes:?}");
19377 assert!(
19378 st.events
19379 .iter()
19380 .any(|event| event.message.contains("waiting for"))
19381 );
19382 assert!(st.resource_wait.is_none());
19383 hold(&cache).release();
19385 }
19386
19387 #[tokio::test]
19388 async fn verification_cache_wait_expires_without_starting_commands() {
19389 crate::run::pin_test_home();
19390 let tmp = tempfile::tempdir().unwrap();
19391 let cache = tmp.path().join("target");
19392 let _held = hold(&cache);
19393 let mut st = state(0);
19394 let wait = repository_wait(&st, None);
19395 let (outcomes, _) = with_cache_lease(
19396 &mut st,
19397 Some(&cache),
19398 "gate",
19399 "gate",
19400 tmp.path(),
19401 "abc1234",
19402 Duration::from_secs(60),
19403 "test gate",
19404 &wait,
19405 |_, _| async { panic!("commands must not start without the cache lease") },
19406 )
19407 .await
19408 .unwrap();
19409 assert_eq!(outcomes.len(), 1);
19410 assert!(outcomes[0].resource_blocked);
19411 assert!(st.resource_wait.is_none());
19412 }
19413
19414 #[tokio::test]
19415 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19416 crate::run::pin_test_home();
19417 let tmp = tempfile::tempdir().unwrap();
19418 let cache = tmp.path().join("target-a");
19419 let guard = hold(&cache);
19420 tokio::spawn(async move {
19422 tokio::time::sleep(Duration::from_millis(1500)).await;
19423 guard.release();
19424 });
19425 let mut st = state(60);
19426 let results = run(
19427 vec![job(
19428 sh("alpha", "echo ok"),
19429 tmp.path(),
19430 Duration::from_secs(1),
19431 )],
19432 Arc::new(Semaphore::new(1)),
19433 Some(&cache),
19434 &mut st,
19435 )
19436 .await;
19437 assert!(
19438 matches!(results[0].2, AgentOutcome::Ok(_)),
19439 "the wait must not be charged to the agent"
19440 );
19441 assert!(st.resource_block.is_none());
19442 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19443 assert!(st.resource_wait.is_none() && st.active.is_empty());
19444 }
19445
19446 #[tokio::test]
19447 #[cfg(unix)]
19448 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19449 use std::os::fd::AsRawFd as _;
19450 crate::run::pin_test_home();
19451 let tmp = tempfile::tempdir().unwrap();
19452 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19453 let common = tmp.path().join(".git");
19454 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19455 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19456 assert_eq!(
19457 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19458 0
19459 );
19460 let mut st = state(1);
19461 st.config.coordination = crate::config::test_coordination();
19462 let mut reader = job(
19463 sh("reader", "echo must-not-run"),
19464 tmp.path(),
19465 Duration::from_secs(30),
19466 );
19467 reader.allow_write = false;
19468 let results = run(
19469 vec![reader.clone()],
19470 Arc::new(Semaphore::new(1)),
19471 None,
19472 &mut st,
19473 )
19474 .await;
19475 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19476 assert_eq!(results[0].1.turns, 0);
19477 assert!(FailClass::of(&results[0].2).is_none());
19478 assert_eq!(
19479 st.resource_block.as_ref().unwrap().kind,
19480 "repository coordination"
19481 );
19482 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19483 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19484 drop(external);
19485 let editor = crate::repo_lock::SeatLease::for_checkout(
19486 tmp.path(),
19487 true,
19488 crate::config::test_coordination().as_ref(),
19489 )
19490 .await
19491 .unwrap();
19492 let pause = Pause::new();
19493 let interrupt = Pause::new();
19494 let prompts = Prompts::default();
19495 let ctx = WaveCtx {
19496 run: "r",
19497 node: "review",
19498 prompts: &prompts,
19499 cache: None,
19500 round: None,
19501 carry_seats: false,
19502 pauses: Some((&pause, &interrupt)),
19503 };
19504 let (results, ()) = tokio::join!(
19505 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19506 async {
19507 tokio::time::sleep(Duration::from_millis(50)).await;
19508 pause.park();
19509 }
19510 );
19511 assert!(
19512 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19513 );
19514 assert_eq!(results[0].1.turns, 0);
19515 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19516 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19517 drop(editor);
19518 }
19519
19520 #[tokio::test]
19521 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19522 crate::run::pin_test_home();
19523 let tmp = tempfile::tempdir().unwrap();
19524 let cache = tmp.path().join("target-b");
19525 let _held = hold(&cache);
19526 let mut st = state(1);
19527 let results = run(
19528 vec![job(
19529 sh("alpha", "echo ok"),
19530 tmp.path(),
19531 Duration::from_secs(1),
19532 )],
19533 Arc::new(Semaphore::new(1)),
19534 Some(&cache),
19535 &mut st,
19536 )
19537 .await;
19538 let (_, seat, out) = &results[0];
19539 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19540 assert!(wave_blocked(&results));
19541 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19542 assert_eq!(seat.turns, 0, "nothing was asked");
19543 assert!(
19544 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19545 "no agent was spawned"
19546 );
19547 assert!(FailClass::of(out).is_none(), "not a failure class");
19548 let block = st.resource_block.as_ref().expect("recorded for the resume");
19549 assert_eq!(block.kind, "shared build cache");
19550 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19551 assert!(st.active.is_empty());
19552 }
19553
19554 #[tokio::test]
19555 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19556 crate::run::pin_test_home();
19557 let tmp = tempfile::tempdir().unwrap();
19558 let mut st = state(1);
19559 let results = run(
19560 vec![job(
19561 sh("alpha", "echo ok"),
19562 tmp.path(),
19563 Duration::from_secs(1),
19564 )],
19565 Arc::new(Semaphore::new(0)),
19566 None,
19567 &mut st,
19568 )
19569 .await;
19570 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19571 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19572 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19573 }
19574
19575 #[tokio::test]
19576 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19577 crate::run::pin_test_home();
19578 let tmp = tempfile::tempdir().unwrap();
19579 let mut st = state(60);
19580 let results = run(
19581 vec![job(
19582 sh("alpha", "sleep 30"),
19583 tmp.path(),
19584 Duration::from_secs(1),
19585 )],
19586 Arc::new(Semaphore::new(1)),
19587 None,
19588 &mut st,
19589 )
19590 .await;
19591 let out = &results[0].2;
19592 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19593 let class = FailClass::of(out).expect("a failure");
19594 assert_eq!(class, FailClass::Timeout);
19595 assert!(should_hand_over(None, &class), "existing handover policy");
19596 assert!(st.resource_block.is_none());
19597 }
19598}