1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530enum FixStep {
532 Resource,
533 Recorded { report_lost: bool },
534}
535
536struct FixClaim {
553 path: PathBuf,
554}
555
556impl FixClaim {
557 fn acquire(dir: &Path) -> Result<Self> {
558 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559 let path = dir.join("fix.lock");
560 let retry = crate::lock_file::Retry::new(None);
561 match Self::create(&path, retry) {
562 Ok(claim) => Ok(claim),
563 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564 if Self::reclaim_if_dead(&path) {
565 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566 } else {
567 bail!(
568 "another repair or `magi fix` is already running for this run ({} exists)",
569 path.display()
570 )
571 }
572 }
573 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574 }
575 }
576
577 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578 let mut f = retry.create(path)?;
579 use std::io::Write as _;
580 writeln!(f, "{}", std::process::id())?;
582 Ok(Self {
583 path: path.to_owned(),
584 })
585 }
586
587 fn reclaim_if_dead(path: &Path) -> bool {
591 let dead = std::fs::read_to_string(path)
592 .ok()
593 .and_then(|body| body.trim().parse::<u32>().ok())
594 .is_some_and(|pid| !crate::proc::pid_alive(pid));
595 dead && std::fs::remove_file(path).is_ok()
596 }
597}
598
599impl Drop for FixClaim {
600 fn drop(&mut self) {
601 let _ = std::fs::remove_file(&self.path);
602 }
603}
604
605struct DriverExecution<'a> {
608 runner: &'a mut Runner,
609 driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613 fn drop(&mut self) {
614 self.driver.release_local(&mut self.runner.state);
615 }
616}
617
618impl Runner {
619 pub async fn start(
621 repo: &Path,
622 instruction: String,
623 config: Config,
624 origin: Origin,
625 ) -> Result<Self> {
626 Self::start_naming(repo, instruction, "", config, origin).await
627 }
628
629 pub async fn start_naming(
632 repo: &Path,
633 instruction: String,
634 _also_scan: &str,
635 config: Config,
636 origin: Origin,
637 ) -> Result<Self> {
638 let repo = git::toplevel(repo).await?;
639 let missing = agent::missing_programs(&config.agents);
640 if !missing.is_empty() {
641 bail!(
642 "these agent programs are not on PATH: {}. Fix the roster in \
643 magi.toml or install them.",
644 missing.join(", ")
645 );
646 }
647 let base_branch =
648 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649 .await?;
650 let fetched_base = resolve_base(
651 &repo,
652 &base_branch,
653 &config.merge.remote,
654 config.coordination.as_ref(),
655 )
656 .await?;
657 let base_commit = fetched_base.commit.clone();
658 let roles = config.resolve_roles()?;
659 let max_parallel = config.graph.max_parallel.max(1);
660 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661 state.fetched_base = Some(fetched_base);
665 state.origin = Some(origin);
666 state.reserve()?;
667 state.event("start", format!("run {} created", state.id));
668 state.save()?;
669 Ok(Self {
670 state,
671 roles,
672 sem: Arc::new(Semaphore::new(max_parallel)),
673 pause: Pause::new(),
674 interrupt: Pause::new(),
675 })
676 }
677
678 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692 Self::review_taking_over(repo, branch, config, None, origin).await
693 }
694
695 pub async fn review_taking_over(
702 repo: &Path,
703 branch: &str,
704 config: Config,
705 takeover: Option<crate::handover::Takeover>,
706 origin: Origin,
707 ) -> Result<Self> {
708 let repo = git::toplevel(repo).await?;
709 let missing = agent::missing_programs(&config.agents);
710 if !missing.is_empty() {
711 bail!(
712 "these agent programs are not on PATH: {}. Fix the roster in \
713 magi.toml or install them.",
714 missing.join(", ")
715 );
716 }
717 let base_branch =
718 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719 .await?;
720 if base_branch == branch {
721 bail!("`{branch}` is the base branch; there is nothing to review against");
722 }
723 let fetched_base = resolve_base(
724 &repo,
725 &base_branch,
726 &config.merge.remote,
727 config.coordination.as_ref(),
728 )
729 .await?;
730 let base_commit = fetched_base.commit.clone();
731
732 let roles = config.resolve_roles()?;
733 let max_parallel = config.graph.max_parallel.max(1);
734 let mut state = RunState::new(
735 repo.clone(),
736 base_branch,
737 base_commit.clone(),
738 String::new(),
739 config,
740 );
741 state.fetched_base = Some(fetched_base);
742 state.origin = Some(origin);
743 state.reserve()?;
744
745 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752 earlier: Vec::new(),
753 home: crate::run::home(),
754 choice: None,
755 task_instruction: None,
756 });
757 let wait = repository_wait(&state, None);
758 let coordination_config = state.config.coordination.clone();
759 let coordination =
760 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761 .await?;
762 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763 if let Some(released) = &released {
764 state.event(
765 "release",
766 format!(
767 "took `{branch}` over from run {}: its worktree was released: {}",
768 crate::run::short_of(&released.old_id),
769 released.audit
770 ),
771 );
772 }
773 if let Some(choice) = takeover.choice.as_ref()
777 && let Err(e) =
778 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779 .await
780 {
781 if let Some(released) = &released {
782 released.restore(&repo, branch).await;
783 }
784 return Err(e.context("applying the owner's answer about the diverged branch"));
785 }
786 for id in takeover.earlier.iter().rev() {
792 if let Ok(old) = RunState::load_under(id, &takeover.home)
793 && let Some(head) = old.pr_head.clone()
794 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795 {
796 state.pr_head = Some(head);
797 break;
798 }
799 }
800 drop(coordination);
801 let opened = Self::open_review(
802 &repo,
803 branch,
804 state,
805 roles,
806 max_parallel,
807 base_commit,
808 &takeover,
809 )
810 .await;
811 if opened.is_err()
812 && let Some(released) = &released
813 {
814 let _coordination = crate::repo_lock::SeatLease::operation_with(
815 &repo,
816 coordination_config.as_ref(),
817 &wait,
818 )
819 .await?;
820 released.restore(&repo, branch).await;
821 }
822 opened
823 }
824
825 pub async fn recover_pr(
827 repo: &Path,
828 task: &str,
829 selection: crate::pr_recovery::Selection,
830 mut config: Config,
831 ) -> Result<Self> {
832 selection.configure(&mut config)?;
833 selection.check_task(task, &selection.instruction)?;
834 let base_branch =
835 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836 .await?;
837 let fetched_base = resolve_base(
838 repo,
839 &base_branch,
840 &config.merge.remote,
841 config.coordination.as_ref(),
842 )
843 .await?;
844 let base_commit = fetched_base.commit.clone();
845 selection
846 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847 .await?;
848 Box::pin(selection.verify_source(repo)).await?;
849 if let Some(proof) = &selection.source_proof {
850 anyhow::ensure!(
851 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852 "source rebase base is outside the fetched landing base"
853 );
854 }
855 let roles = config.resolve_roles()?;
856 let max_parallel = config.graph.max_parallel.max(1);
857 let mut state = RunState::new(
858 repo.to_path_buf(),
859 base_branch,
860 base_commit.clone(),
861 selection.instruction.clone(),
862 config,
863 );
864 state.fetched_base = Some(fetched_base);
865 state.origin = Some(crate::run::Origin::queue(task));
866 state.pr_head = Some(selection.branch.clone());
867 state.pr_recovery = Some(selection.clone());
868 state.reserve()?;
869 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870 state.discard_reserved();
872 return Err(e);
873 }
874 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875 }
876
877 async fn open_selected(
878 repo: &Path,
879 state: RunState,
880 roles: ResolvedRoles,
881 max_parallel: usize,
882 base_commit: String,
883 selection: crate::pr_recovery::Selection,
884 ) -> Result<Self> {
885 let mut state = state;
886 let branch = state.branch_for('A');
887 let wait = repository_wait(&state, None);
888 let prepared = async {
889 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890 git::git(repo, &["branch", &branch, selection.source()]).await?;
891 drop(coordination);
892 anyhow::Ok(())
893 }
894 .await;
895 if let Err(e) = prepared {
896 state.discard_reserved();
899 return Err(e);
900 }
901 state.event(
902 "recovery",
903 format!(
904 "selected implementation {} for {} (public lease {})",
905 selection.source(),
906 selection.url,
907 selection.head
908 ),
909 );
910 let takeover = crate::handover::Takeover {
911 earlier: Vec::new(),
912 home: crate::run::home(),
913 choice: None,
914 task_instruction: Some(selection.instruction.clone()),
915 };
916 if let Err(e) = state.save() {
917 state.discard_reserved();
918 return Err(e);
919 }
920 let id = state.id.clone();
921 let opened = Self::open_review(
922 repo,
923 &branch,
924 state,
925 roles,
926 max_parallel,
927 base_commit,
928 &takeover,
929 )
930 .await;
931 match opened {
932 Ok(runner) => Ok(runner),
933 Err(e) => {
934 if let Ok(mut partial) = RunState::load(&id) {
935 partial.status = RunStatus::Blocked;
936 partial.driver_exited = true;
937 partial.event(
938 "recovery",
939 format!("startup failed; retained source and candidate {branch}: {e:#}"),
940 );
941 partial.save()?;
942 }
943 Err(crate::pr_recovery::StartupFailed {
944 run: id,
945 cause: format!("{e:#}"),
946 }
947 .into())
948 }
949 }
950 }
951
952 async fn inherit(
957 repo: &Path,
958 branch: &str,
959 base_commit: &str,
960 takeover: &crate::handover::Takeover,
961 ) -> Option<crate::recovery::RecoveryRecord> {
962 use crate::recovery::{Relation, assess};
963 let newest = takeover.earlier.last()?;
964 let mut chosen = None;
965 for id in takeover.earlier.iter().rev() {
966 if let Ok(old) = RunState::load_under(id, &takeover.home)
967 && !old.reviews.is_empty()
968 && (old.winner().is_some_and(|w| w.branch == branch)
969 || old.pr_head.as_deref() == Some(branch))
970 {
971 chosen = Some(old);
972 break;
973 }
974 }
975 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976 .await
977 .unwrap_or_default();
978 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979 Some(last) if last.head == tip => Relation::Same,
980 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981 _ => Relation::Diverged,
982 };
983 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984 Some(assess(from, chosen.as_ref(), base_commit, relation))
985 }
986
987 async fn open_review(
990 repo: &Path,
991 branch: &str,
992 mut state: RunState,
993 roles: ResolvedRoles,
994 max_parallel: usize,
995 base_commit: String,
996 takeover: &crate::handover::Takeover,
997 ) -> Result<Self> {
998 let wait = repository_wait(&state, None);
999 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000 if state.pr_recovery.is_none() {
1001 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002 }
1003 let start = review_base(
1006 repo,
1007 &state.config.merge.remote,
1008 &state.base_branch,
1009 &base_commit,
1010 branch,
1011 )
1012 .await;
1013 let log = git::log_oneline(repo, &start, branch)
1014 .await
1015 .unwrap_or_default();
1016 let log_text = if log.trim().is_empty() {
1017 "(no commit messages)"
1018 } else {
1019 log.trim()
1020 };
1021 let mut instruction = format!(
1022 "Review the work already on branch `{branch}`. There is no task \
1023 statement: what the change claims to do is whatever its commits \
1024 say.\n\n{log_text}"
1025 );
1026 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030 if let Some(task) = takeover
1031 .task_instruction
1032 .as_deref()
1033 .filter(|t| !t.trim().is_empty())
1034 && rec.reviewed_head.is_some()
1035 {
1036 instruction = format!(
1037 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038 task.trim()
1039 );
1040 rec.criteria_inherited = true;
1041 }
1042 state.event("recovery", rec.summary());
1043 state.recovery = Some(rec);
1044 }
1045 if let Some(selected) = &state.pr_recovery {
1046 instruction = selected.instruction.clone();
1047 }
1048 state.task_instruction = takeover.task_instruction.clone();
1049 state.instruction = instruction;
1050 state.reviewed_commits = Some(
1051 git::subjects(repo, &start, branch)
1052 .await
1053 .unwrap_or_default(),
1054 );
1055
1056 for id in &takeover.earlier {
1059 if let Ok(old) = RunState::load_under(id, &takeover.home)
1060 && old.repo == state.repo
1061 && (old.pr_head.as_deref() == Some(branch)
1062 || old.winner().is_some_and(|w| w.branch == branch))
1063 {
1064 for hold in old.branch_owner_holds {
1065 if !state.branch_owner_holds.contains(&hold) {
1066 state.event(
1067 "approval-policy",
1068 format!(
1069 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070 hold.control.key(),
1071 hold.file,
1072 hold.head
1073 ),
1074 );
1075 state.branch_owner_holds.push(hold);
1076 }
1077 }
1078 }
1079 }
1080 let reviewed_head = git::rev_parse(repo, branch).await?;
1081 state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083 let worktree = state.worktree_root().join("under-review");
1086 if let Some(parent) = worktree.parent() {
1087 tokio::fs::create_dir_all(parent).await.ok();
1088 }
1089 let path = worktree.to_string_lossy().to_string();
1090 git::git(repo, &["worktree", "add", &path, branch])
1091 .await
1092 .with_context(|| {
1093 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094 })?;
1095
1096 let setup = match ensure_setup_config(&mut state, repo).await {
1097 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098 Err(e) => Err(e),
1099 };
1100 if let Err(e) = setup {
1101 git::worktree_remove_clean(repo, &worktree).await.ok();
1102 return Err(e);
1103 }
1104 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105 .await
1106 .unwrap_or(0);
1107 if commits == 0 {
1108 git::worktree_remove_clean(repo, &worktree).await.ok();
1109 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110 }
1111 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112 .await
1113 .map(|f| f.len())
1114 .unwrap_or(0);
1115 if files == 0
1116 && let (Ok(head_tree), Ok(base_tree)) = (
1117 git::tree_of(&worktree, "HEAD").await,
1118 git::tree_of(&worktree, &base_commit).await,
1119 )
1120 && head_tree == base_tree
1121 {
1122 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123 git::worktree_remove_clean(repo, &worktree).await.ok();
1124 bail!(
1125 "`{branch}` at {} has a tree identical to base {}; this usually means \
1126 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127 against `{}/{branch}`) rather than an empty change",
1128 short(&head),
1129 short(&base_commit),
1130 state.config.merge.remote
1131 );
1132 }
1133 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134 .await
1135 .unwrap_or_default();
1136
1137 state.candidates.push(Candidate {
1138 index: 0,
1139 label: 'A',
1140 agent: EXISTING_BRANCH.to_owned(),
1143 branch: branch.to_owned(),
1144 worktree,
1145 summary: String::new(),
1146 stat,
1147 files,
1148 commits,
1149 empty: false,
1150 failed: None,
1151 verified_noop: None,
1152 duration_ms: 0,
1153 folded: false,
1154 });
1155 state.tally = Some(Tally {
1156 first_choice: BTreeMap::from([('A', 0)]),
1157 borda: BTreeMap::new(),
1158 winner: 'A',
1159 rankings: 0,
1160 unanimous_initial: false,
1161 deliberated: false,
1162 changed_votes: 0,
1163 unanimous_final: false,
1164 tie_break: None,
1165 judges: 0,
1169 present: 0,
1170 quorum: 0,
1171 met_quorum: true,
1172 uncontested: Some("review-only run: nothing competed".to_owned()),
1173 });
1174 state.status = RunStatus::Reviewing;
1175 state.prep_complete = Some(true);
1176 state.event(
1177 "start",
1178 format!(
1179 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180 state.id
1181 ),
1182 );
1183 state.save()?;
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub fn resume(id: &str) -> Result<Self> {
1195 let state = RunState::load(id)?;
1196 crate::pr_head::ensure_owner(&state)?;
1197 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198 bail!(
1199 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200 );
1201 }
1202 if let Some(to) = &state.released_to {
1203 bail!(
1204 "run {} cannot be resumed: its worktree was released to run {}",
1205 state.short(),
1206 crate::run::short_of(to)
1207 );
1208 }
1209 let roles = state.config.resolve_roles()?;
1210 if let Some(id) = &state.config.graph.selected_implementer
1211 && state.candidates.iter().any(|c| c.agent != *id)
1212 {
1213 bail!(
1214 "replacement implementer pin does not match the saved candidates; task must stay held"
1215 );
1216 }
1217 let max_parallel = state.config.graph.max_parallel.max(1);
1218 Ok(Self {
1219 state,
1220 roles,
1221 sem: Arc::new(Semaphore::new(max_parallel)),
1222 pause: Pause::new(),
1223 interrupt: Pause::new(),
1224 })
1225 }
1226
1227 pub async fn execute(&mut self) -> Result<()> {
1234 if let Some(selection) = &self.state.pr_recovery
1235 && (selection.source_proof.is_some() || selection.source_run.is_some())
1236 {
1237 Box::pin(selection.verify_source(&self.state.repo)).await?;
1238 }
1239 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242 || self.state.land_approval.is_some()
1243 || self.state.merge_decision.is_some()
1244 || !self.state.merge_repairs.is_empty()
1245 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246 {
1247 let claim = FixClaim::acquire(&self.state.dir())?;
1248 let disk = RunState::load(&self.state.id)?;
1249 let checkpoint = |s: &RunState| {
1250 (
1251 s.merge_repairs.len(),
1252 s.merge_repairs
1253 .last()
1254 .map(|r| (r.question.clone(), r.phase)),
1255 s.land_approval
1256 .as_ref()
1257 .map(|a| (a.question.clone(), a.head.clone())),
1258 )
1259 };
1260 if checkpoint(&disk) != checkpoint(&self.state) {
1261 bail!(
1262 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263 );
1264 }
1265 Some(claim)
1266 } else {
1267 None
1268 };
1269 let mut execution = self.claim_driver().await?;
1270 let runner = &mut *execution.runner;
1271 let wait = runner.repository_wait();
1272 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275 let result = match result {
1276 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277 record_repository_block(&mut runner.state, &e);
1278 runner.park_here()?;
1279 Ok(())
1280 }
1281 other => other,
1282 };
1283 execution.driver.finish(&mut runner.state).await?;
1284 let ended = if result.is_err() {
1285 Some({
1286 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1287 if let Err(error) = &result {
1288 notice.message = format!(
1289 "Run {} stopped with an error: {}",
1290 runner.state.id,
1291 error.to_string().lines().next().unwrap_or("unknown error")
1292 );
1293 }
1294 notice
1295 })
1296 } else {
1297 crate::notices::run_ended(&runner.state)
1298 };
1299 if let Some(notice) = ended {
1300 crate::notices::raise(notice);
1301 }
1302 result
1303 }
1304
1305 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1306 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1307 }
1308
1309 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1310 let wait = self.repository_wait();
1311 coordinate_operation(&mut self.state, cwd, &wait).await
1312 }
1313
1314 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1316 let owner = match crate::pr_head::owner_lock_wait(
1317 &crate::run::home(),
1318 &self.state.id,
1319 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1320 )
1321 .await
1322 {
1323 Ok(owner) => owner,
1324 Err(error) => {
1325 let detail = format!("worktree maintenance or ownership update: {error:#}");
1326 self.state.resource_block = Some(crate::run::ResourceBlock {
1330 cause: crate::run::ResourceBlockCause::Other,
1331 node: "start".into(),
1332 kind: "worktree maintenance".into(),
1333 detail: detail.clone(),
1334 at: Timestamp::now(),
1335 });
1336 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1337 }
1338 };
1339 let disk = RunState::load(&self.state.id)?;
1340 crate::pr_head::ensure_owner(&disk)?;
1341 let claimed =
1342 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1343 anyhow::ensure!(
1348 disk.liveness(claimed) != crate::run::Liveness::Live,
1349 "run already has a live driver; wait for its execution to end"
1350 );
1351 if let Some(to) = &disk.released_to {
1352 bail!(
1353 "run {} cannot continue: its worktree was released to run {}",
1354 self.state.short(),
1355 crate::run::short_of(to)
1356 );
1357 }
1358 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1359 let mut execution = DriverExecution {
1360 runner: self,
1361 driver,
1362 };
1363 let _owner = owner;
1366 let state = &mut execution.runner.state;
1367 state.parked = false;
1372 state.resource_wait = None;
1373 state.resource_block = None;
1374 state.clear_active();
1381 let pid = std::process::id();
1396 state.driver_pid = Some(pid);
1397 state.driver_started_at = crate::proc::process_started_at(pid);
1398 state.driver_exited = false;
1399 state.driver_claimed = Some(true);
1400 state.begin_pass();
1401 execution.driver.bind(state);
1402 state.save()?;
1403 execution.driver.record(state)?;
1404 Ok(execution)
1405 }
1406
1407 async fn execute_graph(&mut self) -> Result<()> {
1408 self.state.configure_recovery_resume()?;
1409 self.persist_candidates().await?;
1412 if self
1413 .state
1414 .merge_decision
1415 .as_ref()
1416 .is_some_and(|decision| !decision.closed)
1417 && self.state.status != RunStatus::Landing
1418 {
1419 if crate::merge_decision::resumable_hold(&self.state) {
1420 self.state.merge = Some(MergeOutcome {
1421 mode: MergeMode::Pr,
1422 ok: false,
1423 detail: self
1424 .state
1425 .merge_decision
1426 .as_ref()
1427 .unwrap()
1428 .target
1429 .pr
1430 .clone(),
1431 empty: false,
1432 });
1433 self.run_land().await?;
1434 self.settle_questions();
1435 return Ok(());
1436 }
1437 bail!(
1438 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1439 );
1440 }
1441 if self.state.config.graph.selected_implementer.is_some()
1442 && self.state.status == RunStatus::Blocked
1443 && self.state.tally.is_none()
1444 && self.state.candidate_preservation.is_empty()
1445 && self.state.interrupted_implementations.is_empty()
1446 {
1447 bail!(
1448 "selected implementation is held; explicitly release the owning task to retry it"
1449 );
1450 }
1451
1452 if self.state.merge_repairs.last().is_some_and(|r| {
1453 matches!(
1454 r.phase,
1455 crate::run::MergeRepairPhase::Requested
1456 | crate::run::MergeRepairPhase::Fixing
1457 | crate::run::MergeRepairPhase::Reviewing
1458 )
1459 }) {
1460 if self.repair_merge_review().await? {
1461 self.run_land().await?;
1462 }
1463 self.settle_questions();
1464 return Ok(());
1465 }
1466 if self.state.pr_publication_pending {
1469 self.merge().await?;
1470 if self.resource_halt()? {
1471 return Ok(());
1472 }
1473 self.state.save()?;
1474 self.settle_questions();
1475 return Ok(());
1476 }
1477 if self.state.status == RunStatus::Stalled {
1490 if self.recover_stall().await? {
1491 self.finish_after_tally().await?;
1492 } else {
1493 self.state.save()?;
1495 }
1496 return Ok(());
1497 }
1498 if self.state.status == RunStatus::Landing
1508 || crate::handover::landing_checkpoint(&self.state)
1509 {
1510 self.state.land_sync_pending = true;
1511 self.state.save()?;
1512 self.run_land().await?;
1513 self.settle_questions();
1518 return Ok(());
1519 }
1520 if self.state.github_text.is_some()
1525 && self.state.merge.is_none()
1526 && self.state.status == RunStatus::Gating
1527 {
1528 self.merge().await?;
1529 if self.resource_halt()? {
1530 return Ok(());
1531 }
1532 self.state.save()?;
1533 return Ok(());
1534 }
1535 if let Err(e) = self.prep().await {
1536 if e.is::<crate::repo_lock::ResourceBlocked>() {
1537 return Err(e);
1538 }
1539 self.state.status = RunStatus::Blocked;
1540 self.state
1541 .prep_failure
1542 .get_or_insert_with(|| format!("{e:#}"));
1543 self.state
1544 .event("prep", format!("preparation stopped: {e:#}"));
1545 self.state.save()?;
1546 return Err(e);
1547 }
1548 if self.park_here()? {
1549 return Ok(());
1550 }
1551 self.advise().await?;
1552 if self.park_here()? {
1553 return Ok(());
1554 }
1555 self.implement().await?;
1556 if self.park_here()? {
1557 return Ok(());
1558 }
1559 if self.state.status == RunStatus::VerifiedNoop {
1563 return Ok(());
1564 }
1565 self.judge().await?;
1566 if self.park_here()? {
1567 return Ok(());
1568 }
1569 self.deliberate().await?;
1570 if self.park_here()? {
1571 return Ok(());
1572 }
1573 self.vote().await?;
1574 if self.park_here()? {
1575 return Ok(());
1576 }
1577 self.tally()?;
1578 if self.state.status == RunStatus::Stalled {
1583 self.state.save()?;
1587 return Ok(());
1588 }
1589 self.finish_after_tally().await?;
1590 Ok(())
1591 }
1592
1593 fn stop_for_resource(&mut self) -> Result<()> {
1598 if self.pause.parked() || self.interrupt.parked() {
1601 self.park_here()?;
1602 return Ok(());
1603 }
1604 if self.state.resource_block.is_some() {
1605 self.state.status = RunStatus::Blocked;
1606 }
1607 self.state.save()
1608 }
1609
1610 fn resource_halt(&mut self) -> Result<bool> {
1612 if self.state.resource_block.is_none() {
1613 return Ok(false);
1614 }
1615 self.stop_for_resource()?;
1616 Ok(true)
1617 }
1618
1619 fn park_here(&mut self) -> Result<bool> {
1626 let parking = self.pause.parked() || self.interrupt.parked();
1627 if !parking && self.state.resource_block.is_some() {
1628 self.stop_for_resource()?;
1629 return Ok(true);
1630 }
1631 if !self.pause.parked() && !self.interrupt.parked() {
1636 return Ok(false);
1637 }
1638 if self.state.parked {
1639 return Ok(true);
1640 }
1641 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1642 if let Some(block) = &mut self.state.resource_block
1643 && matches!(
1644 block.cause,
1645 crate::run::ResourceBlockCause::Cancelled { .. }
1646 )
1647 {
1648 block.cause = crate::run::ResourceBlockCause::Cancelled {
1649 park: cause.clone(),
1650 };
1651 }
1652 let why = match cause.as_ref().map(ParkCause::label) {
1653 Some(reason) => format!(
1654 "parked after `{}` ({reason}) — resume to carry on from here",
1655 self.state.status.as_str()
1656 ),
1657 None => format!(
1658 "parked after `{}` — resume to carry on from here",
1659 self.state.status.as_str()
1660 ),
1661 };
1662 self.state.event("park", why);
1663 self.state
1664 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1665 self.state.parked = true;
1666 self.state.save()?;
1667 Ok(true)
1668 }
1669
1670 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1672 if self.state.prep_failure.is_none() {
1673 return Ok(());
1674 }
1675 let failed_at = self
1676 .state
1677 .events
1678 .iter()
1679 .rev()
1680 .find(|event| event.node == "prep")
1681 .map_or(self.state.updated_at, |event| event.at);
1682 anyhow::ensure!(
1683 approval_at > failed_at,
1684 "preparation retry requires a new explicit release after the failure"
1685 );
1686 self.state.prep_failure = None;
1687 self.state.prep_complete = Some(false);
1688 self.state
1689 .event("prep-retry", "operator approved retry of preparation");
1690 Ok(())
1691 }
1692
1693 pub fn on_pause(&mut self, pause: Pause) {
1695 self.pause = pause;
1696 }
1697
1698 pub fn watch_interrupt(&mut self, pause: Pause) {
1704 self.interrupt = pause;
1705 }
1706
1707 fn settle_questions(&mut self) {
1727 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1728 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1729 }
1730 }
1731
1732 async fn finish_after_tally(&mut self) -> Result<()> {
1735 self.graft().await?;
1738 if self.resource_halt()? {
1739 return Ok(());
1740 }
1741 self.fold_losers().await?;
1742 self.sync_to_base().await?;
1747 if self.resource_halt()? {
1748 return Ok(());
1749 }
1750 if self.state.status == RunStatus::AlreadyInBase {
1751 return Ok(());
1752 }
1753 self.review_loop().await?;
1754 if self.resource_halt()? {
1755 return Ok(());
1756 }
1757 self.sync_to_base().await?;
1758 if self.resource_halt()? {
1759 return Ok(());
1760 }
1761 if self.state.status == RunStatus::AlreadyInBase {
1762 return Ok(());
1763 }
1764 self.gate().await?;
1765 self.merge().await?;
1766 if self.resource_halt()? {
1767 return Ok(());
1768 }
1769 self.state.save()?;
1770 Ok(())
1771 }
1772
1773 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1780 let result = match ensure_setup_config(&mut self.state, repo).await {
1781 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1782 Err(e) => Err(e),
1783 };
1784 if let Err(e) = result {
1785 git::worktree_remove_clean(repo, wt).await.ok();
1786 if in_prep {
1787 self.rollback_prep(repo).await;
1788 }
1789 self.state.status = RunStatus::Blocked;
1790 self.state.event(
1791 "setup",
1792 format!("worktree setup failed in {}: {e:#}", wt.display()),
1793 );
1794 self.state.save()?;
1795 return Err(e);
1796 }
1797 Ok(())
1798 }
1799
1800 async fn rollback_prep(&mut self, repo: &Path) {
1802 let root = self.state.worktree_root();
1803 for c in std::mem::take(&mut self.state.candidates) {
1804 if matches!(
1805 git::worktree_remove_clean(repo, &c.worktree).await,
1806 Ok(true)
1807 ) {
1808 git::branch_delete(repo, &c.branch).await.ok();
1809 } else {
1810 self.state.candidates.push(c);
1811 }
1812 }
1813 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1814 for j in 1..=seats {
1815 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1816 .await
1817 .ok();
1818 }
1819 for k in 1..=self.state.config.graph.advisors {
1820 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1821 .await
1822 .ok();
1823 }
1824 }
1825
1826 async fn prep(&mut self) -> Result<()> {
1827 if let Some(reason) = &self.state.prep_failure {
1828 bail!(
1829 "candidate preparation previously failed: {reason}; correct the source before retrying"
1830 );
1831 }
1832 if self
1833 .state
1834 .prep_complete
1835 .unwrap_or(!self.state.candidates.is_empty())
1836 {
1837 return Ok(());
1838 }
1839 let repo = self.state.repo.clone();
1840 let _coordination = self.coordinate(&repo).await?;
1843 let base = self.state.base_commit.clone();
1844 let plan = self.state.sources.plan(&repo, &base).await?;
1845 self.rollback_prep(&repo).await;
1846 if !self.state.candidates.is_empty() {
1847 bail!(
1848 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1849 );
1850 }
1851 self.state.prep_complete = Some(false);
1852 self.state.status = RunStatus::Prep;
1853 self.state.save()?;
1854 let repo = self.state.repo.clone();
1855 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1856 let root = self.state.worktree_root();
1857 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1858
1859 let hooks_dir = self.state.dir().join("hooks");
1862 if self.state.config.blind.commit_msg_hook {
1863 std::fs::create_dir_all(&hooks_dir)
1864 .with_context(|| format!("create {}", hooks_dir.display()))?;
1865 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1866 let path = hooks_dir.join("commit-msg");
1867 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1868 make_executable(&path)?;
1869 git::acquire_worktree_config(&repo).await?;
1877 self.state.enabled_worktree_config = true;
1878 }
1879
1880 ensure_setup_config(&mut self.state, &repo).await?;
1883
1884 for (index, (spec, label)) in self
1885 .roles
1886 .implementers
1887 .clone()
1888 .into_iter()
1889 .zip(labels)
1890 .enumerate()
1891 {
1892 let branch = self.state.branch_for(label);
1893 let worktree = root.join(format!("cand-{label}"));
1894 self.state.event(
1898 "prep-intent",
1899 serde_json::json!({
1900 "branch": branch,
1901 "worktree": worktree,
1902 "start": start,
1903 "picks": plan.picks,
1904 })
1905 .to_string(),
1906 );
1907 self.state.save()?;
1908 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1909 if self.state.config.blind.commit_msg_hook {
1910 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1911 }
1912 git::local_exclude(&worktree, "/.magi/").await?;
1913 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1914 git::worktree_remove_clean(&repo, &worktree).await.ok();
1917 git::branch_delete(&repo, &branch).await.ok();
1918 self.rollback_prep(&repo).await;
1919 self.state.status = RunStatus::Blocked;
1920 self.state
1921 .event("prep", format!("worktree setup failed: {e:#}"));
1922 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1923 self.state.save()?;
1924 return Err(e);
1925 }
1926 self.state.candidates.push(Candidate {
1927 index,
1928 label,
1929 agent: spec.id.clone(),
1930 branch,
1931 worktree,
1932 summary: String::new(),
1933 stat: String::new(),
1934 files: 0,
1935 commits: 0,
1936 empty: false,
1937 failed: None,
1938 verified_noop: None,
1939 duration_ms: 0,
1940 folded: false,
1941 });
1942 self.state.save()?;
1943 for pick in &plan.picks {
1944 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1945 {
1946 self.state.status = RunStatus::Blocked;
1947 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1948 self.state
1949 .event("prep", format!("cannot apply referenced commit: {e}"));
1950 self.state.save()?;
1951 return Err(e);
1952 }
1953 }
1954 }
1955
1956 for j in 1..=self.roles.judges.len() {
1957 let wt = root.join(format!("judge-{j}"));
1958 if !wt.exists() {
1959 git::worktree_add_detached(&repo, &wt, &base).await?;
1960 self.setup_seat_worktree(&repo, &wt, true).await?;
1961 }
1962 }
1963
1964 if self.state.config.graph.advise {
1972 for k in 1..=self.state.config.graph.advisors {
1973 let wt = root.join(format!("advisor-{k}"));
1974 if !wt.exists() {
1975 git::worktree_add_detached(&repo, &wt, &base).await?;
1976 self.setup_seat_worktree(&repo, &wt, true).await?;
1977 }
1978 }
1979 }
1980
1981 let authors: Vec<&str> = self
1986 .roles
1987 .implementers
1988 .iter()
1989 .map(|a| a.id.as_str())
1990 .collect();
1991 let overlap: Vec<String> = self
1992 .roles
1993 .judges
1994 .iter()
1995 .enumerate()
1996 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1997 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1998 .collect();
1999 if !overlap.is_empty() {
2000 let note = format!(
2001 "{} also authored a candidate; blind, but the panel is less \
2002 independent than {} distinct agents would be",
2003 overlap.join(", "),
2004 self.roles.judges.len()
2005 );
2006 self.state.event("prep", note);
2007 }
2008
2009 self.state.event(
2010 "prep",
2011 format!(
2012 "{} candidates, {} judges, base {} ({})",
2013 self.state.candidates.len(),
2014 self.roles.judges.len(),
2015 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2016 self.state.base_branch
2017 ),
2018 );
2019 self.state.prep_complete = Some(true);
2020 self.state.status = RunStatus::Implementing;
2021 self.state.save()?;
2022 Ok(())
2023 }
2024
2025 async fn advise(&mut self) -> Result<()> {
2058 let implement_untouched = self
2059 .state
2060 .candidates
2061 .iter()
2062 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2063 if !self.state.config.graph.advise || self.state.advise_attempted {
2064 return Ok(());
2065 }
2066 if !implement_untouched {
2067 self.state.event(
2068 "advise",
2069 "skipping the design-deliberation stage: at least one \
2070 candidate already shows implementation progress, so this \
2071 run is past the point the stage exists to run before"
2072 .to_owned(),
2073 );
2074 self.state.advise_attempted = true;
2075 self.state.save()?;
2076 return Ok(());
2077 }
2078 let run_id = self.state.id.clone();
2079 let prompts = self.state.config.prompts.clone();
2080 let instruction = self.state.instruction.clone();
2081 let language = self.state.config.graph.language.clone();
2082 let root = self.state.worktree_root();
2083 let n = self.state.config.graph.advisors;
2084 let where_recorded = self.state.dir().join("run.json");
2085
2086 let seats = match self.state.config.advisors() {
2087 Ok(seats) if !seats.is_empty() => seats,
2088 Ok(_) => {
2089 self.state.event(
2090 "advise",
2091 format!(
2092 "[graph] advisors is 0; skipping the design-deliberation \
2093 stage and continuing without a synthesis brief (see {})",
2094 where_recorded.display()
2095 ),
2096 );
2097 self.state.advise_attempted = true;
2098 self.state.save()?;
2099 return Ok(());
2100 }
2101 Err(e) => {
2102 self.state.event(
2103 "advise",
2104 format!(
2105 "could not resolve advisor seats ({e:#}); continuing \
2106 without a design-deliberation brief (see {})",
2107 where_recorded.display()
2108 ),
2109 );
2110 self.state.advise_attempted = true;
2111 self.state.save()?;
2112 return Ok(());
2113 }
2114 };
2115
2116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2117 let artifacts = agent::artifacts_dir(&self.state.dir());
2118 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2119
2120 let mut jobs = Vec::new();
2121 for (i, spec) in seats.iter().cloned().enumerate() {
2122 let seat_key = format!("advisor-{}", i + 1);
2123 let seat = self.seat(&seat_key, &spec.id);
2124 jobs.push(SeatJob {
2125 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2126 spec,
2127 seat,
2128 cwd: worktrees[i % worktrees.len()].clone(),
2129 timeout,
2130 allow_write: false,
2131 sessions: false,
2132 artifacts: artifacts.clone(),
2133 stem: seat_key,
2134 handover: None,
2135 });
2136 }
2137
2138 self.state.event(
2139 "advise",
2140 format!(
2141 "{} advisor seat(s) sketching a design in parallel",
2142 jobs.len()
2143 ),
2144 );
2145 let mut quota_losses = Vec::new();
2146 let cache = self.state.config.cache_dir();
2147 let ctx = WaveCtx {
2148 carry_seats: false,
2149 pauses: Some((&self.pause, &self.interrupt)),
2150 run: &run_id,
2151 node: "advise",
2152 prompts: &prompts,
2153 cache: cache.as_deref(),
2154 round: None,
2155 };
2156 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2157 let results = ask_json_wave::<Proposal>(
2158 jobs,
2159 Arc::clone(&self.sem),
2160 self.state.config.graph.retries,
2161 &advisor_roster,
2162 &ctx,
2163 &mut quota_losses,
2164 &mut self.state,
2165 &|p: &Proposal| p.validate(),
2166 )
2167 .await;
2168 if self.state.resource_block.is_some() {
2169 return self.stop_for_resource();
2170 }
2171 self.state.quota.extend(quota_losses);
2172
2173 let mut records = Vec::with_capacity(results.len());
2174 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2175 let agent_id = seat.agent.clone();
2176 self.state.seats.insert(seat.key.clone(), seat);
2177 match res {
2178 Ok((proposal, out)) => {
2179 self.state
2180 .event("advise", format!("advisor-{} proposed a design", i + 1));
2181 records.push(advise::AdvisorRecord::proposed(
2182 i + 1,
2183 agent_id,
2184 proposal,
2185 out.duration_ms,
2186 ));
2187 }
2188 Err(e) => {
2189 self.state.event(
2190 "advise",
2191 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2192 );
2193 records.push(advise::AdvisorRecord::failed(
2194 i + 1,
2195 agent_id,
2196 e.to_string(),
2197 ));
2198 }
2199 }
2200 }
2201
2202 let mut advice = advise::Advice {
2203 records,
2204 synthesis: None,
2205 };
2206 if advice.proposals().is_empty() {
2207 self.state.event(
2208 "advise",
2209 "no advisor produced a usable proposal; continuing without a \
2210 synthesis brief"
2211 .to_owned(),
2212 );
2213 } else {
2214 match self
2215 .synthesize_brief(
2216 &advice,
2217 &instruction,
2218 &language,
2219 &worktrees[0],
2220 &artifacts,
2221 &run_id,
2222 &prompts,
2223 cache.as_deref(),
2224 )
2225 .await
2226 {
2227 Ok(Some(text)) => {
2228 self.state.event(
2229 "advise",
2230 "synthesized a design brief for the implementer".to_owned(),
2231 );
2232 advice.synthesis = Some(text);
2233 }
2234 Ok(None) => {
2235 self.state.event(
2236 "advise",
2237 "the synthesis seat produced nothing usable; continuing \
2238 without a design brief"
2239 .to_owned(),
2240 );
2241 }
2242 Err(e) => {
2243 if e.is::<crate::repo_lock::ResourceBlocked>() {
2244 return Err(e);
2245 }
2246 self.state.event(
2247 "advise",
2248 format!("could not synthesize a design brief: {e:#}"),
2249 );
2250 }
2251 }
2252 }
2253 advise::apply_reflection(&mut advice);
2254
2255 self.state.advice = Some(advice);
2256 self.state.advise_attempted = true;
2257 self.state.save()?;
2258 Ok(())
2259 }
2260
2261 #[allow(clippy::too_many_arguments)]
2273 async fn synthesize_brief(
2274 &mut self,
2275 advice: &advise::Advice,
2276 instruction: &str,
2277 language: &str,
2278 cwd: &Path,
2279 artifacts: &Path,
2280 run_id: &str,
2281 prompts: &Prompts,
2282 cache: Option<&Path>,
2283 ) -> Result<Option<String>> {
2284 let chain = agent::pick_chain(
2285 &self.state.config.agents,
2286 self.state.config.roles.synthesizer.as_ref(),
2287 &agent::installed,
2288 "synthesizer",
2289 )?;
2290 let proposals = advice.proposals();
2291 let mut prompt = prompt::with_overlay(
2292 prompt::synthesize_brief(instruction, &proposals, language),
2293 prompts.overlay("advise"),
2294 );
2295 if cache.is_some() {
2296 prompt.push('\n');
2301 prompt.push_str(&prompt::build_cache_note("advise", false));
2302 }
2303 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2304 let mut last = None;
2309 for (n, spec) in chain.iter().enumerate() {
2310 if n > 0 {
2311 self.state
2312 .event("advise", format!("synthesis falling back to {}", spec.id));
2313 }
2314 let mut seat = self.seat("advise-synthesis", &spec.id);
2315 let wait = self.repository_wait();
2316 let outcome = wait
2317 .scope(agent::invoke(
2318 spec,
2319 &mut seat,
2320 &Invocation {
2321 scope: crate::agent::InvocationScope::Repository,
2322 coordination: self.state.config.coordination.as_ref(),
2323 fetched_base: None,
2324 cwd,
2325 prompt: &prompt,
2326 timeout,
2327 allow_write: false,
2328 unsandboxed: false,
2329 sessions: false,
2330 artifacts,
2331 stem: &if n == 0 {
2332 "advise-synthesis".to_owned()
2333 } else {
2334 format!("advise-synthesis-{}", spec.id)
2335 },
2336 run: run_id,
2337 node: "advise",
2338 cache_dir: None,
2339 attachments: &[],
2340 writable: &[],
2341 },
2342 ))
2343 .await;
2344 if outcome.is_ok() {
2345 self.state.seats.insert(seat.key.clone(), seat);
2346 }
2347 let advance = agent::chain_advances(&outcome);
2348 last = Some(outcome);
2349 if !advance {
2350 break;
2351 }
2352 }
2353 let out = last.expect("a chain holds at least one agent")?;
2356 if !out.usable() {
2357 return Ok(None);
2358 }
2359 let text =
2360 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2361 Ok((!text.trim().is_empty()).then_some(text))
2362 }
2363
2364 async fn implement(&mut self) -> Result<()> {
2367 self.persist_candidates().await?;
2368 let run_id = self.state.id.clone();
2373 let prompts = self.state.config.prompts.clone();
2374 let todo: Vec<usize> = self
2375 .state
2376 .candidates
2377 .iter()
2378 .enumerate()
2379 .filter(|(_, c)| {
2380 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2381 && c.failed.is_none()
2382 && !c.empty
2383 })
2384 .map(|(i, _)| i)
2385 .collect();
2386 if todo.is_empty() {
2387 return self.after_implement();
2388 }
2389 self.state.status = RunStatus::Implementing;
2390
2391 let language = self.state.config.graph.language.clone();
2392 let github_language = self.state.config.graph.github_language.clone();
2393 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2394 let sessions = self.state.config.graph.sessions;
2395 let artifacts = agent::artifacts_dir(&self.state.dir());
2396 let brief = self
2400 .state
2401 .advice
2402 .as_ref()
2403 .and_then(|a| a.synthesis.as_deref())
2404 .map(str::to_owned);
2405 let attachments = self.state.attachments.clone();
2406
2407 let mut jobs = Vec::new();
2408 for &i in &todo {
2409 let (index, label, worktree) = {
2410 let c = &self.state.candidates[i];
2411 (c.index, c.label, c.worktree.clone())
2412 };
2413 let spec = self.roles.implementers[index].clone();
2414 let seat_key = format!("impl-{label}");
2415 let seat = self.seat(&seat_key, &spec.id);
2416 let instruction = seeded_instruction(&self.state);
2417 jobs.push(SeatJob {
2418 spec,
2419 seat,
2420 prompt: prompt::implement_in(
2421 &instruction,
2422 &worktree.to_string_lossy(),
2423 &language,
2424 &github_language,
2425 brief.as_deref(),
2426 &attachments,
2427 ),
2428 cwd: worktree,
2429 timeout,
2430 allow_write: true,
2431 sessions,
2432 artifacts: artifacts.clone(),
2433 stem: format!("impl-{label}"),
2434 handover: None,
2435 });
2436 }
2437
2438 self.state.event(
2439 "implement",
2440 format!("{} candidates in parallel", jobs.len()),
2441 );
2442 let mut sent = jobs.clone();
2448 let cache = self.state.config.cache_dir();
2449 let ctx = WaveCtx {
2450 carry_seats: false,
2451 pauses: Some((&self.pause, &self.interrupt)),
2452 run: &run_id,
2453 node: "implement",
2454 prompts: &prompts,
2455 cache: cache.as_deref(),
2456 round: None,
2457 };
2458 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2459 if !wave_blocked(&results) {
2460 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2461 .await;
2462 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2463 .await?;
2464 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2465 .await;
2466 }
2467 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2468
2469 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2470 && results
2471 .iter()
2472 .any(|(_, _, out)| FailClass::of(out).is_some());
2473 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2474 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2475 if self
2479 .state
2480 .candidate_preservation
2481 .contains_key(&self.state.candidates[i].label)
2482 {
2483 self.state.seats.insert(seat.key.clone(), seat);
2484 }
2485 continue;
2486 }
2487 let seat_key = seat.key.clone();
2488 let agent = seat.agent.clone();
2497 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2498 self.state.seats.insert(seat.key.clone(), seat);
2499 let label = self.state.candidates[i].label;
2500 self.state.interrupted_implementations.remove(&label);
2501
2502 let (summary, duration, failed, verified_claim) = match out {
2503 AgentOutcome::Ok(o) => {
2504 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2505 let failed = (!o.usable()).then(|| {
2506 if o.timed_out {
2507 "agent timed out".to_owned()
2508 } else {
2509 format!(
2510 "agent exited with {}",
2511 crate::notices::exit_status(o.exit_code)
2512 )
2513 }
2514 });
2515 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2516 (text, o.duration_ms, failed, verified_claim)
2517 }
2518 AgentOutcome::Dropped(o) => {
2524 let why = o
2525 .dropped
2526 .as_ref()
2527 .map(|d| d.why.as_str())
2528 .unwrap_or("the CLI ended the stream without delivering its answer");
2529 (
2530 String::new(),
2531 o.duration_ms,
2532 Some(format!("the CLI dropped the stream ({why})")),
2533 None,
2534 )
2535 }
2536 AgentOutcome::Quota(o) => {
2537 self.state.quota.push(QuotaLoss {
2538 seat: seat_key,
2539 node: "implement".to_owned(),
2540 at: Timestamp::now(),
2541 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2542 });
2543 (
2544 String::new(),
2545 o.duration_ms,
2546 Some("rate limited (quota); produced no change".to_owned()),
2547 None,
2548 )
2549 }
2550 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2551 (String::new(), 0, Some(e), None)
2552 }
2553 };
2554
2555 let c = &mut self.state.candidates[i];
2558 if !exhausted_the_fallback_chain {
2559 c.agent = agent;
2560 }
2561 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2562 c.duration_ms = duration;
2563 c.failed = failed;
2564 c.verified_noop = verified_claim;
2565 self.state
2566 .candidate_preservation
2567 .insert(label, "awaiting persistence".to_owned());
2568 }
2569 self.state.save()?;
2570 self.persist_candidates().await?;
2571
2572 if resource_blocked {
2573 return self.stop_for_resource();
2574 }
2575 if selected_failed {
2576 self.state.status = RunStatus::Blocked;
2577 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2578 self.state.save()?;
2579 bail!(
2580 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2581 );
2582 }
2583 self.after_implement()
2584 }
2585
2586 async fn persist_candidates(&mut self) -> Result<()> {
2588 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2589 for label in labels {
2590 let i = self
2591 .state
2592 .candidates
2593 .iter()
2594 .position(|c| c.label == label)
2595 .context("pending candidate is missing")?;
2596 let c = self.state.candidates[i].clone();
2597 let interrupted = self.state.interrupted_implementations.contains(&label);
2598 let result = async {
2599 let wait = crate::repo_lock::WaitPolicy::new(
2600 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2601 || false,
2602 );
2603 let _coordination =
2604 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2605 if let Err(e) = git::snapshot_uncommitted(
2608 &c.worktree,
2609 &self
2610 .state
2611 .dir()
2612 .join("artifacts")
2613 .join(format!("candidate-{label}-recovery")),
2614 )
2615 .await
2616 {
2617 self.state.event(
2618 "preserve",
2619 format!("candidate {label} snapshot incomplete: {e:#}"),
2620 );
2621 }
2622 let rescued = git::rescue_commit(
2623 &c.worktree,
2624 &format!("magi: candidate {label} (uncommitted work)"),
2625 )
2626 .await?;
2627 self.state.note_withheld("implement", &rescued.withheld);
2628 let base = &self.state.base_commit;
2629 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2630 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2631 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2632 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2633 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2634 if c.empty && commits > 0 && !patch.trim().is_empty()
2635 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2636 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2637 || self.state.tally.is_some())
2638 {
2639 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2643 }
2644 let c = &mut self.state.candidates[i];
2645 c.commits = commits;
2646 c.stat = stat;
2647 c.files = files;
2648 let no_changes = commits == 0 || patch.trim().is_empty();
2649 c.empty = no_changes && !interrupted;
2650 if interrupted && no_changes {
2651 c.failed = None;
2652 c.verified_noop = None;
2653 }
2654 if !c.empty {
2655 c.failed = None;
2656 c.verified_noop = None;
2657 }
2658 self.state.event(
2659 "preserve",
2660 format!("candidate {label}: {files} files, {commits} commits"),
2661 );
2662 Ok::<_, anyhow::Error>(())
2663 }
2664 .await;
2665 match result {
2666 Ok(()) => {
2667 self.state.candidate_preservation.remove(&label);
2668 if !interrupted || self.state.candidates[i].files > 0 {
2669 self.state.interrupted_implementations.remove(&label);
2670 }
2671 }
2672 Err(e) => {
2673 let reason = format!("candidate {label} preservation failed: {e:#}");
2674 self.state.event("preserve", &reason);
2675 self.state.candidate_preservation.insert(label, reason);
2676 }
2677 }
2678 if self.state.candidate_preservation.is_empty()
2679 && self.state.status == RunStatus::Blocked
2680 && self.state.candidates.iter().all(|c| c.failed.is_none())
2681 {
2682 self.state.status = RunStatus::Implementing;
2686 self.state.event(
2687 "preserve",
2688 "candidate persistence recovered; implementation can continue",
2689 );
2690 }
2691 self.state.save()?;
2692 }
2693 if !self.state.candidate_preservation.is_empty() {
2694 self.state.status = RunStatus::Blocked;
2695 self.state.save()?;
2696 bail!(
2697 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2698 );
2699 }
2700 Ok(())
2701 }
2702
2703 async fn resume_undelivered(
2731 &mut self,
2732 results: &mut [(usize, SeatState, AgentOutcome)],
2733 sent: &[SeatJob],
2734 prompts: &Prompts,
2735 run_id: &str,
2736 ) {
2737 for (wi, seat, out) in results.iter_mut() {
2738 let Some(dropped) = (match &*out {
2739 AgentOutcome::Dropped(o) => o.dropped.clone(),
2740 _ => None,
2741 }) else {
2742 continue;
2743 };
2744 let Some(job) = sent.get(*wi) else { continue };
2745 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2747 self.state.event(
2748 "implement",
2749 format!(
2750 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2751 work is in the tree",
2752 seat.key, dropped.output_tokens, dropped.why
2753 ),
2754 );
2755 continue;
2756 }
2757 if !has_context(&job.spec, seat, job.sessions) {
2765 self.state.event(
2766 "implement",
2767 format!(
2768 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2769 is no session left to resume",
2770 seat.key, dropped.output_tokens, dropped.why
2771 ),
2772 );
2773 continue;
2774 }
2775 self.state.event(
2776 "implement",
2777 format!(
2778 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2779 conversation",
2780 seat.key, dropped.output_tokens, dropped.why
2781 ),
2782 );
2783 let mut retry = job.clone();
2784 retry.seat = seat.clone();
2785 retry.prompt = prompt::resume_after_drop(&dropped.why);
2786 retry.timeout = retry_budget(job.timeout, true);
2787 retry.stem = format!("{}-resume", job.stem);
2788 let cache = self.state.config.cache_dir();
2789 let ctx = WaveCtx {
2790 carry_seats: false,
2791 pauses: Some((&self.pause, &self.interrupt)),
2792 run: run_id,
2793 node: "implement",
2794 prompts,
2795 cache: cache.as_deref(),
2796 round: None,
2797 };
2798 let (resumed_seat, resumed) =
2799 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2800 *seat = resumed_seat;
2801 *out = resumed;
2802 }
2803 }
2804
2805 async fn resume_seat_handovers(
2870 &mut self,
2871 results: &mut [(usize, SeatState, AgentOutcome)],
2872 sent: &mut [SeatJob],
2873 prompts: &Prompts,
2874 run_id: &str,
2875 ) -> Result<()> {
2876 let instruction = seeded_instruction(&self.state);
2877 let language = self.state.config.graph.language.clone();
2878 let github_language = self.state.config.graph.github_language.clone();
2879 let brief = self
2880 .state
2881 .advice
2882 .as_ref()
2883 .and_then(|a| a.synthesis.as_deref())
2884 .map(str::to_owned);
2885 let attachments = self.state.attachments.clone();
2886 for (wi, seat, out) in results.iter_mut() {
2887 let others: BTreeSet<String> = sent
2890 .iter()
2891 .enumerate()
2892 .filter(|(j, _)| j != wi)
2893 .map(|(_, j)| j.spec.id.clone())
2894 .collect();
2895 let Some(job) = sent.get_mut(*wi) else {
2896 continue;
2897 };
2898 let start = self
2903 .roles
2904 .implementer_roster
2905 .iter()
2906 .position(|s| s.id == job.spec.id)
2907 .unwrap_or(0);
2908 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2909 let mut fallback_attempt = 0usize;
2910 let mut prev: Option<FailClass> = None;
2911 while let Some(cur) = FailClass::of(&*out) {
2912 if !should_hand_over(prev.as_ref(), &cur) {
2913 break;
2914 }
2915 let Some(next) =
2916 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2917 .cloned()
2918 else {
2919 break;
2920 };
2921 tried.insert(next.id.clone());
2922 fallback_attempt += 1;
2923
2924 let coordination = self.coordinate(&job.cwd).await?;
2925 if let Ok(r) = git::rescue_commit(
2926 &job.cwd,
2927 &format!(
2928 "magi: candidate {} (uncommitted work before {} fallback)",
2929 seat.key,
2930 if cur == FailClass::Quota {
2931 "quota"
2932 } else {
2933 "handover"
2934 }
2935 ),
2936 )
2937 .await
2938 {
2939 self.state.note_withheld("implement", &r.withheld);
2940 }
2941 drop(coordination);
2942
2943 record_handover(
2944 &mut self.state,
2945 "implement",
2946 &seat.key,
2947 &seat.agent,
2948 &next.id,
2949 &cur,
2950 &fail_reason(&*out),
2951 );
2952 prev = Some(cur.clone());
2953
2954 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2955 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2956 job.spec = next.clone();
2964 let mut retry = job.clone();
2965 retry.seat = new_seat;
2966 retry.prompt = prompt::implement_in(
2967 &instruction,
2968 &job.cwd.to_string_lossy(),
2969 &language,
2970 &github_language,
2971 brief.as_deref(),
2972 &attachments,
2973 );
2974 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2975 let cache = self.state.config.cache_dir();
2976 let ctx = WaveCtx {
2977 carry_seats: false,
2978 pauses: Some((&self.pause, &self.interrupt)),
2979 run: run_id,
2980 node: "implement",
2981 prompts,
2982 cache: cache.as_deref(),
2983 round: None,
2984 };
2985 let (fallback_seat, fallback_out) = run_one(
2986 retry,
2987 Arc::clone(&self.sem),
2988 &ctx,
2989 &mut self.state,
2990 fallback_attempt,
2991 )
2992 .await;
2993 *seat = fallback_seat;
2994 *out = fallback_out;
2995 }
2996 }
2997 Ok(())
2998 }
2999
3000 async fn resume_unconfirmed_commands(
3024 &mut self,
3025 results: &mut [(usize, SeatState, AgentOutcome)],
3026 sent: &[SeatJob],
3027 prompts: &Prompts,
3028 run_id: &str,
3029 ) {
3030 for (wi, seat, out) in results.iter_mut() {
3031 let AgentOutcome::Ok(o) = &*out else {
3032 continue;
3033 };
3034 if !has_unconfirmed_command(&o.commands) {
3035 continue;
3036 }
3037 let Some(job) = sent.get(*wi) else { continue };
3038 if !has_context(&job.spec, seat, job.sessions) {
3039 self.state.event(
3040 "implement",
3041 format!(
3042 "{}: the reply named a command whose own CLI never confirmed the exit \
3043 status of, but there is no session left to resume",
3044 seat.key
3045 ),
3046 );
3047 continue;
3048 }
3049 self.state.event(
3050 "implement",
3051 format!(
3052 "{}: the reply named a command whose own CLI never confirmed the exit \
3053 status of; resuming the conversation",
3054 seat.key
3055 ),
3056 );
3057 let mut retry = job.clone();
3058 retry.seat = seat.clone();
3059 retry.prompt = prompt::resume_incomplete(
3060 "a command in your last reply had no confirmed exit status",
3061 );
3062 retry.timeout = retry_budget(job.timeout, true);
3063 retry.stem = format!("{}-confirm", job.stem);
3064 let cache = self.state.config.cache_dir();
3065 let ctx = WaveCtx {
3066 carry_seats: false,
3067 pauses: Some((&self.pause, &self.interrupt)),
3068 run: run_id,
3069 node: "implement",
3070 prompts,
3071 cache: cache.as_deref(),
3072 round: None,
3073 };
3074 let (resumed_seat, resumed) =
3075 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3076 *seat = resumed_seat;
3077 *out = resumed;
3078 }
3079 }
3080
3081 async fn continue_fix_report(
3102 &mut self,
3103 mut seat: SeatState,
3104 parse_err: String,
3105 job: &SeatJob,
3106 prompts: &Prompts,
3107 run_id: &str,
3108 round: usize,
3109 ) -> (
3110 SeatState,
3111 Option<FixReport>,
3112 Option<String>,
3113 ContinuationRecord,
3114 ) {
3115 let mut last_err = parse_err;
3116 let mut cumulative_wait_ms = 0u64;
3117 let mut attempts = 0usize;
3118 loop {
3119 if !has_context(&job.spec, &seat, job.sessions) {
3120 self.state.event(
3121 "fix",
3122 format!(
3123 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3124 session left to resume into"
3125 ),
3126 );
3127 let outcome = if attempts == 0 {
3128 ContinuationOutcome::NoSession
3129 } else {
3130 ContinuationOutcome::Exhausted
3131 };
3132 return (
3133 seat,
3134 None,
3135 Some(format!("unparsable fix report: {last_err}")),
3136 ContinuationRecord {
3137 attempts,
3138 cumulative_wait_ms,
3139 outcome,
3140 },
3141 );
3142 }
3143 if attempts >= MAX_FIX_CONTINUATIONS {
3144 self.state.event(
3145 "fix",
3146 format!(
3147 "round {round}: fixer's reply still had no adoption report after \
3148 {attempts} continuation(s) ({last_err}); giving up"
3149 ),
3150 );
3151 return (
3152 seat,
3153 None,
3154 Some(format!(
3155 "unparsable fix report after {attempts} continuation(s): {last_err}"
3156 )),
3157 ContinuationRecord {
3158 attempts,
3159 cumulative_wait_ms,
3160 outcome: ContinuationOutcome::Exhausted,
3161 },
3162 );
3163 }
3164 attempts += 1;
3165 self.state.event(
3166 "fix",
3167 format!(
3168 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3169 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3170 ),
3171 );
3172 let mut retry = job.clone();
3173 retry.seat = seat.clone();
3174 retry.prompt = prompt::resume_incomplete(&last_err);
3175 retry.timeout = retry_budget(job.timeout, true);
3176 retry.stem = format!("{}-continue{attempts}", job.stem);
3177 let cache = self.state.config.cache_dir();
3178 let ctx = WaveCtx {
3179 carry_seats: false,
3180 pauses: Some((&self.pause, &self.interrupt)),
3181 run: run_id,
3182 node: "fix",
3183 prompts,
3184 cache: cache.as_deref(),
3185 round: Some(round),
3186 };
3187 let (resumed_seat, resumed_out) = run_one(
3188 retry,
3189 Arc::clone(&self.sem),
3190 &ctx,
3191 &mut self.state,
3192 attempts,
3193 )
3194 .await;
3195 seat = resumed_seat;
3196 match resumed_out {
3197 AgentOutcome::Ok(o) => {
3198 cumulative_wait_ms += o.duration_ms;
3199 match verdict::extract_json::<FixReport>(&o.text) {
3200 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3201 self.state.event(
3202 "fix",
3203 format!(
3204 "round {round}: fixer's adoption report recovered after \
3205 {attempts} continuation(s)"
3206 ),
3207 );
3208 return (
3209 seat,
3210 Some(report),
3211 None,
3212 ContinuationRecord {
3213 attempts,
3214 cumulative_wait_ms,
3215 outcome: ContinuationOutcome::Resumed,
3216 },
3217 );
3218 }
3219 Ok(_) => {
3227 last_err = "the reply parsed, but it reported a command whose own CLI \
3228 never confirmed an exit status"
3229 .to_owned();
3230 }
3231 Err(e) => last_err = e.to_string(),
3232 }
3233 }
3234 AgentOutcome::Quota(o) => {
3235 cumulative_wait_ms += o.duration_ms;
3236 self.state.quota.push(QuotaLoss {
3237 seat: seat.key.clone(),
3238 node: "fix".to_owned(),
3239 at: Timestamp::now(),
3240 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3241 });
3242 self.state.event(
3243 "fix",
3244 format!(
3245 "round {round}: continuation rate limited (quota); not retrying now"
3246 ),
3247 );
3248 return (
3249 seat,
3250 None,
3251 Some("rate limited (quota) while recovering the fix report".to_owned()),
3252 ContinuationRecord {
3253 attempts,
3254 cumulative_wait_ms,
3255 outcome: ContinuationOutcome::QuotaLost,
3256 },
3257 );
3258 }
3259 AgentOutcome::Dropped(o) => {
3260 cumulative_wait_ms += o.duration_ms;
3261 let why = o
3262 .dropped
3263 .as_ref()
3264 .map(|d| d.why.as_str())
3265 .unwrap_or("the CLI ended the stream without delivering its answer");
3266 last_err = format!("the CLI dropped the stream ({why})");
3267 }
3268 AgentOutcome::Failed(e) => last_err = e,
3269 AgentOutcome::ResourceBlocked(e) => {
3272 return (
3273 seat,
3274 None,
3275 Some(format!("the continuation was not started: {e}")),
3276 ContinuationRecord {
3277 attempts: attempts.saturating_sub(1),
3278 cumulative_wait_ms,
3279 outcome: ContinuationOutcome::NoSession,
3280 },
3281 );
3282 }
3283 }
3284 }
3285 }
3286
3287 fn after_implement(&mut self) -> Result<()> {
3288 if !self.state.candidate_preservation.is_empty() {
3289 bail!("candidate preservation incomplete; judging is unsafe");
3290 }
3291 if self.state.leaks.is_empty() {
3293 let cfg = self.state.config.blind.clone();
3294 let mut leaks = Vec::new();
3295 for c in &self.state.candidates {
3296 let Some(patch) =
3297 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3298 else {
3299 continue;
3300 };
3301 leaks.extend(blind::scan(
3302 &format!("candidate {} patch", c.label),
3303 &patch,
3304 &cfg.vendor_tokens,
3305 ));
3306 }
3307 if !leaks.is_empty() {
3308 let summary = leaks
3309 .iter()
3310 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3311 .collect::<Vec<_>>()
3312 .join(", ");
3313 match cfg.on_leak {
3314 LeakPolicy::Fail => {
3315 self.state.status = RunStatus::Failed;
3316 self.state
3317 .event("blind", format!("vendor text in a patch: {summary}"));
3318 self.state.leaks = leaks;
3319 self.state.save()?;
3320 self.settle_questions();
3321 bail!(
3322 "blind.on_leak = \"fail\" and vendor text reached a \
3323 judged patch: {summary}"
3324 );
3325 }
3326 LeakPolicy::Redact => self.state.event(
3327 "blind",
3328 format!("redacting vendor text for judging: {summary}"),
3329 ),
3330 LeakPolicy::Warn => self.state.event(
3331 "blind",
3332 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3333 ),
3334 }
3335 self.state.leaks = leaks;
3336 }
3337 }
3338
3339 if self.state.viable().is_empty() {
3340 if self.state.all_candidates_verified_noop() {
3341 self.state.status = RunStatus::VerifiedNoop;
3352 self.state.save()?;
3353 self.settle_questions();
3354 return Ok(());
3355 }
3356 self.state.status = RunStatus::Failed;
3357 self.state.save()?;
3358 self.settle_questions();
3359 bail!("no candidate produced a change; nothing to judge");
3360 }
3361 self.state.status = RunStatus::Judging;
3362 self.state.save()?;
3363 Ok(())
3364 }
3365
3366 async fn judge(&mut self) -> Result<()> {
3369 let run_id = self.state.id.clone();
3374 let prompts = self.state.config.prompts.clone();
3375 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3376 return Ok(());
3377 }
3378 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3379 if viable.len() == 1 {
3380 self.state.judge_skipped = true;
3387 self.state.event(
3388 "judge",
3389 format!(
3390 "only candidate {} produced a change; judging skipped",
3391 viable[0].label
3392 ),
3393 );
3394 self.state.save()?;
3395 return Ok(());
3396 }
3397 self.state.status = RunStatus::Judging;
3398
3399 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3400 let language = self.state.config.graph.language.clone();
3401 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3402 let sessions = self.state.config.graph.sessions;
3403 let artifacts = agent::artifacts_dir(&self.state.dir());
3404 let root = self.state.worktree_root();
3405 let base_short = short(&self.state.base_commit);
3406
3407 let mut jobs = Vec::new();
3408 let mut orders = Vec::new();
3409 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3410 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3411 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3412 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3413 let seat_key = format!("judge-{}", j + 1);
3414 let seat = self.seat(&seat_key, &spec.id);
3415 jobs.push(SeatJob {
3416 prompt: prompt::judge(
3417 &self.state.instruction,
3418 &views,
3419 self.roles.judges.len(),
3420 &base_short,
3421 &language,
3422 ),
3423 spec,
3424 seat,
3425 cwd: root.join(format!("judge-{}", j + 1)),
3426 timeout,
3427 allow_write: false,
3428 sessions,
3429 artifacts: artifacts.clone(),
3430 stem: format!("judge-{}", j + 1),
3431 handover: None,
3432 });
3433 }
3434
3435 self.state.event(
3436 "judge",
3437 format!(
3438 "{} judges ranking {} candidates blind",
3439 jobs.len(),
3440 viable.len()
3441 ),
3442 );
3443 let labels_for_check = labels.clone();
3444 let mut quota_losses = Vec::new();
3445 let cache = self.state.config.cache_dir();
3446 let ctx = WaveCtx {
3447 carry_seats: false,
3448 pauses: Some((&self.pause, &self.interrupt)),
3449 run: &run_id,
3450 node: "judge",
3451 prompts: &prompts,
3452 cache: cache.as_deref(),
3453 round: None,
3454 };
3455 let results = ask_json_wave::<Ranking>(
3456 jobs,
3457 Arc::clone(&self.sem),
3458 self.state.config.graph.retries,
3459 &self.roles.judge_roster,
3460 &ctx,
3461 &mut quota_losses,
3462 &mut self.state,
3463 &move |r: &Ranking| r.validate(&labels_for_check),
3464 )
3465 .await;
3466 if self.state.resource_block.is_some() {
3467 return self.stop_for_resource();
3468 }
3469 self.state.quota.extend(quota_losses);
3470
3471 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3472 let agent_id = seat.agent.clone();
3473 self.state.seats.insert(seat.key.clone(), seat);
3474 let mut record = Judgement {
3475 judge: j + 1,
3476 seat: format!("judge-{}", j + 1),
3477 agent: agent_id,
3478 ranking: Vec::new(),
3479 reasons: BTreeMap::new(),
3480 confidence: None,
3481 strengths: BTreeMap::new(),
3482 order: orders[j].clone(),
3483 failed: None,
3484 duration_ms: 0,
3485 };
3486 match res {
3487 Ok((ranking, out)) => {
3488 record.ranking = ranking.normalized();
3489 record.reasons = ranking.reasons;
3490 record.strengths = ranking.strengths;
3491 record.confidence = ranking.confidence;
3492 record.duration_ms = out.duration_ms;
3493 self.state.event(
3494 "judge",
3495 format!(
3496 "judge {} ranked {}",
3497 j + 1,
3498 record.ranking.iter().collect::<String>()
3499 ),
3500 );
3501 }
3502 Err(e) => {
3503 record.failed = Some(e.to_string());
3504 self.state
3505 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3506 }
3507 }
3508 self.state.judgements.push(record);
3509 self.state.save()?;
3510 }
3511 Ok(())
3512 }
3513
3514 async fn deliberate(&mut self) -> Result<()> {
3517 let run_id = self.state.id.clone();
3522 let prompts = self.state.config.prompts.clone();
3523 if !self.state.deliberation.is_empty() {
3524 return Ok(());
3525 }
3526 let tops: Vec<char> = self
3527 .state
3528 .judgements
3529 .iter()
3530 .filter_map(|j| j.ranking.first().copied())
3531 .collect();
3532 let rounds = self.state.config.graph.deliberate_rounds;
3533 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3534 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3535 self.state.event(
3536 "deliberate",
3537 format!("judges agreed on {} outright; no deliberation", tops[0]),
3538 );
3539 }
3540 self.state.status = RunStatus::Voting;
3541 self.state.save()?;
3542 return Ok(());
3543 }
3544
3545 self.state.status = RunStatus::Deliberating;
3546 self.state.event(
3547 "deliberate",
3548 format!(
3549 "split: first choices were {} — opening {rounds} round(s)",
3550 tops.iter().collect::<String>()
3551 ),
3552 );
3553
3554 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3555 let language = self.state.config.graph.language.clone();
3556 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3557 let sessions = self.state.config.graph.sessions;
3558 let artifacts = agent::artifacts_dir(&self.state.dir());
3559 let root = self.state.worktree_root();
3560 let base_short = short(&self.state.base_commit);
3561
3562 for round in 1..=rounds {
3566 let mut turns: Vec<DeliberationTurn> = Vec::new();
3567 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3568 if self.state.judgements[j].failed.is_some() {
3569 continue;
3570 }
3571 let seat_key = format!("judge-{}", j + 1);
3572 let spec = self.occupant(&seat_key, spec);
3573 let mut seat = self.seat(&seat_key, &spec.id);
3574 let transcript = self.transcript(&turns, j);
3575 let build = |context: Option<&str>| {
3576 prompt::deliberate(
3577 &self.state.instruction,
3578 context,
3579 &transcript,
3580 round,
3581 rounds,
3582 &language,
3583 )
3584 };
3585 let block = self.candidate_block(&viable, &base_short);
3586 let full = build(Some(&block));
3587 let text = if has_context(&spec, &seat, sessions) {
3588 build(None)
3589 } else {
3590 full.clone()
3591 };
3592 let job = SeatJob {
3593 spec,
3594 seat: seat.clone(),
3595 prompt: text,
3596 cwd: root.join(format!("judge-{}", j + 1)),
3597 timeout,
3598 allow_write: false,
3599 sessions,
3600 artifacts: artifacts.clone(),
3601 stem: format!("delib-{round}-judge-{}", j + 1),
3602 handover: Some(full),
3603 };
3604 let cache = self.state.config.cache_dir();
3605 let ctx = WaveCtx {
3606 carry_seats: false,
3607 pauses: Some((&self.pause, &self.interrupt)),
3608 run: &run_id,
3609 node: "deliberate",
3610 prompts: &prompts,
3611 cache: cache.as_deref(),
3612 round: None,
3613 };
3614 let mut losses = Vec::new();
3618 let mut results = ask_wave_with::<String>(
3619 vec![job],
3620 Arc::clone(&self.sem),
3621 0,
3622 &self.roles.judge_roster,
3623 &ctx,
3624 &mut losses,
3625 &mut self.state,
3626 &|text: &str| {
3627 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3628 },
3629 )
3630 .await;
3631 self.state.quota.extend(losses);
3632 let (updated, res, _) = results.pop().expect("one job in, one result out");
3633 seat = updated;
3634 let agent_id = seat.agent.clone();
3635 self.state.seats.insert(seat.key.clone(), seat);
3636 let body = match res {
3637 Ok((body, _)) => body,
3638 Err(e) => {
3641 self.state
3642 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3643 continue;
3644 }
3645 };
3646 let tentative = verdict::extract_json::<Position>(&body)
3647 .ok()
3648 .and_then(|p| p.tentative)
3649 .and_then(|s| s.trim().chars().next())
3650 .map(|c| c.to_ascii_uppercase());
3651 self.state.event(
3652 "deliberate",
3653 format!(
3654 "round {round}: judge {} now favours {}",
3655 j + 1,
3656 tentative.map_or("—".to_owned(), |c| c.to_string())
3657 ),
3658 );
3659 turns.push(DeliberationTurn {
3660 judge: j + 1,
3661 agent: agent_id,
3662 body: blind::sanitize_prose(&body, &self.state.config.blind),
3663 tentative,
3664 });
3665 }
3666 self.state
3667 .deliberation
3668 .push(DeliberationRound { round, turns });
3669 self.state.save()?;
3670 }
3671
3672 self.state.status = RunStatus::Voting;
3673 self.state.save()?;
3674 Ok(())
3675 }
3676
3677 async fn vote(&mut self) -> Result<()> {
3680 let run_id = self.state.id.clone();
3685 let prompts = self.state.config.prompts.clone();
3686 if !self.state.votes.is_empty() {
3687 return Ok(());
3688 }
3689 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3690 if viable.len() == 1 {
3691 return Ok(());
3692 }
3693 self.state.status = RunStatus::Voting;
3694
3695 let language = self.state.config.graph.language.clone();
3696 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3697 let sessions = self.state.config.graph.sessions;
3698 let artifacts = agent::artifacts_dir(&self.state.dir());
3699 let root = self.state.worktree_root();
3700 let base_short = short(&self.state.base_commit);
3701 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3702
3703 let mut jobs = Vec::new();
3704 let mut seats_at = Vec::new();
3705 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3706 if self
3707 .state
3708 .judgements
3709 .get(j)
3710 .is_some_and(|r| r.failed.is_some())
3711 {
3712 continue;
3713 }
3714 let seat_key = format!("judge-{}", j + 1);
3715 let spec = self.occupant(&seat_key, spec);
3716 let seat = self.seat(&seat_key, &spec.id);
3717 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3718 let text = if has_context(&spec, &seat, sessions) {
3719 prompt::final_vote(&viable, &language)
3720 } else {
3721 full.clone()
3722 };
3723 jobs.push(SeatJob {
3724 spec,
3725 seat,
3726 prompt: text,
3727 cwd: root.join(format!("judge-{}", j + 1)),
3728 timeout,
3729 allow_write: false,
3730 sessions,
3731 artifacts: artifacts.clone(),
3732 stem: format!("vote-judge-{}", j + 1),
3733 handover: Some(full),
3734 });
3735 seats_at.push(j);
3736 }
3737
3738 self.state.event(
3739 "vote",
3740 format!(
3741 "collecting {} final votes one by one, privately",
3742 jobs.len()
3743 ),
3744 );
3745 let allowed = viable.clone();
3746 let mut quota_losses = Vec::new();
3747 let cache = self.state.config.cache_dir();
3748 let ctx = WaveCtx {
3749 carry_seats: false,
3750 pauses: Some((&self.pause, &self.interrupt)),
3751 run: &run_id,
3752 node: "vote",
3753 prompts: &prompts,
3754 cache: cache.as_deref(),
3755 round: None,
3756 };
3757 let results = ask_json_wave::<FinalVote>(
3758 jobs,
3759 Arc::clone(&self.sem),
3760 self.state.config.graph.retries,
3761 &self.roles.judge_roster,
3762 &ctx,
3763 &mut quota_losses,
3764 &mut self.state,
3765 &move |v: &FinalVote| match v.label() {
3766 Some(c) if allowed.contains(&c) => Ok(()),
3767 other => bail!("vote {other:?} is not one of {allowed:?}"),
3768 },
3769 )
3770 .await;
3771 if self.state.resource_block.is_some() {
3772 return self.stop_for_resource();
3773 }
3774 self.state.quota.extend(quota_losses);
3775
3776 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3777 let agent_id = seat.agent.clone();
3778 self.state.seats.insert(seat.key.clone(), seat);
3779 let initial = self
3780 .state
3781 .judgements
3782 .get(j)
3783 .and_then(|r| r.ranking.first().copied());
3784 let mut record = VoteRecord {
3785 judge: j + 1,
3786 agent: agent_id,
3787 vote: None,
3788 reason: String::new(),
3789 changed: false,
3790 };
3791 match res {
3792 Ok((v, _)) => {
3793 record.vote = v.label();
3794 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3795 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3796 self.state.event(
3797 "vote",
3798 format!(
3799 "judge {} voted {}{}",
3800 j + 1,
3801 record.vote.unwrap_or('?'),
3802 if record.changed { " (changed)" } else { "" }
3803 ),
3804 );
3805 }
3806 Err(e) => {
3807 self.state
3808 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3809 }
3810 }
3811 self.state.votes.push(record);
3812 self.state.save()?;
3813 }
3814 Ok(())
3815 }
3816
3817 fn tally(&mut self) -> Result<()> {
3820 if self.state.tally.is_some() {
3821 return Ok(());
3822 }
3823 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3824 let tops: Vec<char> = self
3825 .state
3826 .judgements
3827 .iter()
3828 .filter_map(|j| j.ranking.first().copied())
3829 .collect();
3830 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3831
3832 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3835 let mut cast: Vec<char> = Vec::new();
3836 for (i, j) in self.state.judgements.iter().enumerate() {
3837 let vote = self
3838 .state
3839 .votes
3840 .iter()
3841 .find(|v| v.judge == i + 1)
3842 .and_then(|v| v.vote)
3843 .or_else(|| j.ranking.first().copied());
3844 if let Some(v) = vote {
3845 *first_choice.entry(v).or_insert(0) += 1;
3846 cast.push(v);
3847 }
3848 }
3849
3850 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3851 for j in &self.state.judgements {
3852 let n = j.ranking.len();
3853 for (pos, label) in j.ranking.iter().enumerate() {
3854 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3855 }
3856 }
3857
3858 let best = first_choice.values().copied().max().unwrap_or(0);
3859 let mut leaders: Vec<char> = first_choice
3860 .iter()
3861 .filter(|(_, v)| **v == best)
3862 .map(|(k, _)| *k)
3863 .collect();
3864 let mut tie_break = None;
3865 if leaders.len() > 1 {
3866 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3867 let borda_leaders: Vec<char> = leaders
3868 .iter()
3869 .copied()
3870 .filter(|l| borda[l] == top_borda)
3871 .collect();
3872 tie_break = Some(if borda_leaders.len() == 1 {
3873 format!(
3874 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3875 leaders.len()
3876 )
3877 } else {
3878 format!(
3879 "{} way tie on both first-choice votes and Borda points, broken by label order",
3880 leaders.len()
3881 )
3882 });
3883 leaders = borda_leaders;
3884 leaders.sort_unstable();
3885 }
3886 let winner = *leaders
3887 .first()
3888 .or(viable.first())
3889 .context("no candidate to declare a winner from")?;
3890
3891 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3892 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3893 let deliberated = !self.state.deliberation.is_empty();
3894
3895 let quota_seats: std::collections::BTreeSet<&str> =
3899 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3900 let mut present = 0usize;
3901 for (i, j) in self.state.judgements.iter().enumerate() {
3902 if quota_seats.contains(j.seat.as_str()) {
3903 continue;
3904 }
3905 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3906 let voted = self
3907 .state
3908 .votes
3909 .iter()
3910 .any(|v| v.judge == i + 1 && v.vote.is_some());
3911 if ranked || voted {
3912 present += 1;
3913 }
3914 }
3915 let needs_quorum = viable.len() > 1;
3921 let judges_total = if needs_quorum {
3922 self.roles.judges.len()
3923 } else {
3924 0
3925 };
3926 let quorum = if needs_quorum {
3927 judges_total / 2 + 1
3928 } else {
3929 0
3930 };
3931 let met_quorum = !needs_quorum || present >= quorum;
3932 let uncontested = (!needs_quorum).then(|| {
3933 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3934 });
3935
3936 self.state.event(
3937 "tally",
3938 match &uncontested {
3939 Some(reason) => format!("winner {winner} — {reason}"),
3940 None => format!(
3941 "winner {winner} — votes {} | initial {} | {} changed | \
3942 {present}/{judges_total} judges{}",
3943 first_choice
3944 .iter()
3945 .map(|(k, v)| format!("{k}:{v}"))
3946 .collect::<Vec<_>>()
3947 .join(" "),
3948 if unanimous_initial {
3949 "unanimous"
3950 } else {
3951 "split"
3952 },
3953 changed_votes,
3954 if met_quorum {
3955 String::new()
3956 } else {
3957 format!(" — below quorum ({quorum} required)")
3958 },
3959 ),
3960 },
3961 );
3962 if !met_quorum {
3963 self.state.event(
3964 "stall",
3965 format!(
3966 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3967 the run stops here, resumable"
3968 ),
3969 );
3970 }
3971 self.state.tally = Some(Tally {
3972 first_choice,
3973 borda,
3974 winner,
3975 rankings: tops.len(),
3976 unanimous_initial,
3977 deliberated,
3978 changed_votes,
3979 unanimous_final,
3980 tie_break,
3981 judges: judges_total,
3982 present,
3983 quorum,
3984 met_quorum,
3985 uncontested,
3986 });
3987 self.state.status = if met_quorum {
3988 RunStatus::Reviewing
3989 } else {
3990 RunStatus::Stalled
3991 };
3992 self.state.save()?;
3993 Ok(())
3994 }
3995
3996 #[allow(clippy::too_many_lines)]
4017 async fn recover_stall(&mut self) -> Result<bool> {
4018 let run_id = self.state.id.clone();
4023 let prompts = self.state.config.prompts.clone();
4024 let quota_seats: BTreeSet<&str> =
4029 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4030 let absent: Vec<String> = self
4031 .state
4032 .judgements
4033 .iter()
4034 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4035 .map(|j| j.seat.clone())
4036 .collect();
4037 if absent.is_empty() {
4038 return Ok(false);
4039 }
4040 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4041 if viable.len() <= 1 {
4042 return Ok(false);
4043 }
4044 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4045 let language = self.state.config.graph.language.clone();
4046 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4047 let sessions = self.state.config.graph.sessions;
4048 let artifacts = agent::artifacts_dir(&self.state.dir());
4049 let root = self.state.worktree_root();
4050 let base_short = short(&self.state.base_commit);
4051 let candidates: Vec<Candidate> = viable.clone();
4052
4053 let mut positions: Vec<usize> = absent
4055 .iter()
4056 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4057 .collect();
4058 if positions.is_empty() {
4059 return Ok(false);
4060 }
4061 positions.sort_unstable();
4062 positions.dedup();
4063
4064 let mut judge_jobs = Vec::new();
4066 for &j in &positions {
4067 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4068 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4069 let seat_key = format!("judge-{}", j + 1);
4070 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4071 let seat = self.seat(&seat_key, &spec.id);
4072 judge_jobs.push(SeatJob {
4073 spec,
4074 seat,
4075 prompt: prompt::judge(
4076 &self.state.instruction,
4077 &views,
4078 self.roles.judges.len(),
4079 &base_short,
4080 &language,
4081 ),
4082 cwd: root.join(seat_key),
4083 timeout,
4084 allow_write: false,
4085 sessions,
4086 artifacts: artifacts.clone(),
4087 stem: format!("judge-{}-recover", j + 1),
4088 handover: None,
4089 });
4090 }
4091
4092 let labels_for_check = labels.clone();
4093 let mut judge_losses = Vec::new();
4094 let retries = self.state.config.graph.retries;
4095 let cache = self.state.config.cache_dir();
4096 let ctx = WaveCtx {
4097 carry_seats: false,
4098 pauses: Some((&self.pause, &self.interrupt)),
4099 run: &run_id,
4100 node: "judge",
4101 prompts: &prompts,
4102 cache: cache.as_deref(),
4103 round: None,
4104 };
4105 let results = ask_json_wave::<Ranking>(
4106 judge_jobs,
4107 Arc::clone(&self.sem),
4108 retries,
4109 &self.roles.judge_roster,
4110 &ctx,
4111 &mut judge_losses,
4112 &mut self.state,
4113 &move |r: &Ranking| r.validate(&labels_for_check),
4114 )
4115 .await;
4116 if self.state.resource_block.is_some() {
4117 self.stop_for_resource()?;
4118 return Ok(false);
4119 }
4120
4121 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4123 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4124 let agent_id = seat.agent.clone();
4125 self.state.seats.insert(seat.key.clone(), seat);
4126 let record = &mut self.state.judgements[j];
4127 match res {
4128 Ok((ranking, out)) => {
4129 record.agent = agent_id;
4130 record.ranking = ranking.normalized();
4131 record.reasons = ranking.reasons;
4132 record.strengths = ranking.strengths;
4133 record.confidence = ranking.confidence;
4134 record.failed = None;
4135 record.duration_ms = out.duration_ms;
4136 recovered.insert(j);
4137 self.state.event(
4138 "recover",
4139 format!("judge {} ranked again after the limit", j + 1),
4140 );
4141 }
4142 Err(e) => {
4143 self.state
4144 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4145 }
4146 }
4147 }
4148
4149 let mut vote_jobs = Vec::new();
4151 let mut vote_pos: Vec<usize> = Vec::new();
4152 for &j in &recovered {
4153 let seat_key = format!("judge-{}", j + 1);
4154 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4155 let seat = self.seat(&seat_key, &spec.id);
4156 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4157 let text = if has_context(&spec, &seat, sessions) {
4158 prompt::final_vote(&labels, &language)
4159 } else {
4160 full.clone()
4161 };
4162 vote_jobs.push(SeatJob {
4163 spec,
4164 seat,
4165 prompt: text,
4166 cwd: root.join(seat_key),
4167 timeout,
4168 allow_write: false,
4169 sessions,
4170 artifacts: artifacts.clone(),
4171 stem: format!("vote-judge-{}-recover", j + 1),
4172 handover: Some(full),
4173 });
4174 vote_pos.push(j);
4175 }
4176 let allowed = labels.clone();
4177 let mut vote_losses = Vec::new();
4178 let vote_retries = self.state.config.graph.retries;
4179 let vote_cache = self.state.config.cache_dir();
4180 let ctx = WaveCtx {
4181 carry_seats: false,
4182 pauses: Some((&self.pause, &self.interrupt)),
4183 run: &run_id,
4184 node: "vote",
4185 prompts: &prompts,
4186 cache: vote_cache.as_deref(),
4187 round: None,
4188 };
4189 let votes = ask_json_wave::<FinalVote>(
4190 vote_jobs,
4191 Arc::clone(&self.sem),
4192 vote_retries,
4193 &self.roles.judge_roster,
4194 &ctx,
4195 &mut vote_losses,
4196 &mut self.state,
4197 &move |v: &FinalVote| match v.label() {
4198 Some(c) if allowed.contains(&c) => Ok(()),
4199 other => bail!("vote {other:?} is not one of {allowed:?}"),
4200 },
4201 )
4202 .await;
4203 if self.state.resource_block.is_some() {
4204 self.stop_for_resource()?;
4205 return Ok(false);
4206 }
4207 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4208 let agent_id = seat.agent.clone();
4209 self.state.seats.insert(seat.key.clone(), seat);
4210 match res {
4211 Ok((v, _)) => {
4212 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4213 rec.vote = v.label();
4214 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4215 } else {
4216 self.state.votes.push(VoteRecord {
4217 judge: j + 1,
4218 agent: agent_id,
4219 vote: v.label(),
4220 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4221 changed: false,
4222 });
4223 }
4224 self.state.event(
4225 "recover",
4226 format!("judge {} voted again after the limit", j + 1),
4227 );
4228 }
4229 Err(e) => {
4230 self.state
4231 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4232 }
4233 }
4234 }
4235
4236 let recovered_keys: BTreeSet<String> = recovered
4240 .iter()
4241 .map(|&j| format!("judge-{}", j + 1))
4242 .collect();
4243 self.state
4244 .quota
4245 .retain(|q| !recovered_keys.contains(&q.seat));
4246 for loss in judge_losses.into_iter().chain(vote_losses) {
4250 if recovered_keys.contains(&loss.seat) {
4251 continue;
4252 }
4253 self.state.quota.retain(|q| q.seat != loss.seat);
4254 self.state.quota.push(loss);
4255 }
4256
4257 self.state.tally = None;
4259 self.tally()?;
4260 Ok(self
4261 .state
4262 .tally
4263 .as_ref()
4264 .map(|t| t.met_quorum)
4265 .unwrap_or(false))
4266 }
4267
4268 async fn graft(&mut self) -> Result<()> {
4284 let Err(e) = self.graft_inner().await else {
4285 return Ok(());
4286 };
4287 if e.is::<crate::repo_lock::ResourceBlocked>() {
4288 return Err(e);
4289 }
4290 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4291 self.state.status = RunStatus::Blocked;
4292 self.state.event("graft", format!("blocked: {e:#}"));
4293 self.state.save()?;
4294 return Err(e);
4295 }
4296 if let Some(rec) = self.state.graft.clone()
4301 && !rec.outcome.settled()
4302 {
4303 if let Some(winner) = self.state.winner().cloned() {
4304 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4305 if re.is::<crate::repo_lock::ResourceBlocked>() {
4306 return Err(re);
4307 }
4308 self.state.status = RunStatus::Blocked;
4309 self.state.event("graft", format!("blocked: {re:#}"));
4310 self.state.save()?;
4311 return Err(re);
4312 }
4313 }
4314 if let Some(g) = self.state.graft.as_mut() {
4315 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4316 }
4317 }
4318 self.state.event("graft", format!("skipped: {e:#}"));
4319 if let Err(e) = self.state.save() {
4320 tracing::warn!("save after graft: {e:#}");
4321 }
4322 Ok(())
4323 }
4324
4325 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4328 let _coordination = self.coordinate(wt).await?;
4329 crate::recovery::restore_repair(wt, pre)
4330 .await
4331 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4332 }
4333
4334 fn graft_skipped(&mut self, why: &str) {
4336 let message = format!("skipped: {why}");
4337 if !self
4338 .state
4339 .events
4340 .iter()
4341 .any(|e| e.node == "graft" && e.message == message)
4342 {
4343 self.state.event("graft", message);
4344 }
4345 }
4346
4347 async fn graft_inner(&mut self) -> Result<()> {
4348 if let Some(rec) = self.state.graft.clone()
4351 && !rec.outcome.settled()
4352 {
4353 if self.state.reviews.is_empty()
4357 && let Some(winner) = self.state.winner().cloned()
4358 {
4359 self.graft_restore(&winner.worktree, &rec.pre).await?;
4360 }
4361 if rec.outcome == GraftOutcome::WaitingForRestore {
4362 self.state.graft = None;
4363 self.state.save()?;
4364 } else {
4365 if let Some(g) = self.state.graft.as_mut() {
4366 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4367 }
4368 self.state.event(
4369 "graft",
4370 "rolled back: the previous round was interrupted; the winner is untouched",
4371 );
4372 self.state.save()?;
4373 return Ok(());
4374 }
4375 }
4376 let plan = match graft::plan(&self.state) {
4377 Ok(plan) => plan,
4378 Err(skip) => {
4379 if skip != graft::Skip::AlreadyDone {
4380 self.graft_skipped(skip.reason());
4381 self.state.save()?;
4382 }
4383 return Ok(());
4384 }
4385 };
4386 let Some(winner) = self.state.winner().cloned() else {
4387 return Ok(());
4388 };
4389 let wt = winner.worktree.clone();
4390 if !git::is_clean(&wt).await? {
4391 self.graft_skipped("the winner's worktree has uncommitted changes");
4392 self.state.save()?;
4393 return Ok(());
4394 }
4395 let pre = git::rev_parse(&wt, "HEAD").await?;
4396 self.state.graft = Some(GraftRecord {
4399 pre: pre.clone(),
4400 post: None,
4401 lines: None,
4402 limit: None,
4403 agent: None,
4404 outcome: GraftOutcome::Running,
4405 requested: graft::requested(&plan),
4406 adoption: None,
4407 adoption_error: None,
4408 });
4409 self.state.event(
4410 "graft",
4411 format!(
4412 "porting strengths of candidate(s) {} onto {}",
4413 plan.strengths
4414 .iter()
4415 .map(|(l, _, _)| l.to_string())
4416 .collect::<Vec<_>>()
4417 .join(", "),
4418 plan.winner
4419 ),
4420 );
4421 self.state.save()?;
4422
4423 let fix_prompt = prompt::graft(
4424 &self.state.instruction,
4425 plan.winner,
4426 &plan.strengths,
4427 &self.state.config.graph.language,
4428 );
4429 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4430 let sessions = self.state.config.graph.sessions;
4431 let artifacts = agent::artifacts_dir(&self.state.dir());
4432 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4433 let (_, seat, out) = self
4434 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4435 SeatJob {
4436 prompt: fix_prompt.clone(),
4437 spec,
4438 seat,
4439 cwd: wt.clone(),
4440 timeout,
4441 allow_write: true,
4442 sessions,
4443 artifacts: artifacts.clone(),
4444 stem: "graft".to_owned(),
4445 handover: None,
4446 }
4447 })
4448 .await;
4449 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4450 self.state.graft.as_mut().expect("graft record").outcome =
4453 GraftOutcome::WaitingForRestore;
4454 return self.stop_for_resource();
4455 }
4456 let mut why: Option<String> = None;
4457 match &out {
4458 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4459 why = Some("the fixer produced nothing usable".to_owned());
4460 }
4461 AgentOutcome::Ok(_) => {}
4462 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4463 AgentOutcome::Quota(o) => {
4464 self.state.quota.push(QuotaLoss {
4465 seat: seat.key.clone(),
4466 node: "graft".to_owned(),
4467 at: Timestamp::now(),
4468 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4469 });
4470 why = Some("rate limited (quota)".to_owned());
4471 }
4472 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4473 why = Some(format!(
4474 "the fixer failed: {}",
4475 e.lines()
4476 .next()
4477 .unwrap_or("")
4478 .chars()
4479 .take(160)
4480 .collect::<String>()
4481 ));
4482 }
4483 }
4484 let agent_id = seat.agent.clone();
4485 self.state.seats.insert(seat.key.clone(), seat);
4486
4487 let mut post = None;
4488 let mut lines = None;
4489 let _coordination = self.coordinate(&wt).await?;
4490 if why.is_none() {
4491 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4492 if let Ok(r) = &rescued {
4493 self.state.note_withheld("graft", &r.withheld);
4494 }
4495 let head = git::rev_parse(&wt, "HEAD").await?;
4496 let dirty = match &rescued {
4500 Ok(r) => git::status_porcelain(&wt)
4501 .await
4502 .map(|st| {
4503 st.lines().any(|l| {
4504 !r.withheld
4505 .iter()
4506 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4507 })
4508 })
4509 .unwrap_or(true),
4510 Err(_) => true,
4511 };
4512 if dirty {
4513 why = Some("the fixer left changes that could not be committed".to_owned());
4514 } else if head == pre {
4515 why = Some("the fixer changed nothing".to_owned());
4516 } else {
4517 lines = git::changed_lines(&wt, &pre, &head).await;
4518 post = Some(head);
4519 }
4520 }
4521 let (adoption, adoption_error) = match &out {
4522 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4523 _ => (None, None),
4524 };
4525 let requested = graft::requested(&plan);
4526 let outcome = match why {
4527 None => {
4528 self.state.event(
4529 "graft",
4530 format!(
4531 "applied: {}",
4532 lines.map_or_else(
4533 || "changed lines unavailable".to_owned(),
4534 |n| format!("{n} changed lines")
4535 )
4536 ),
4537 );
4538 GraftOutcome::Applied
4539 }
4540 Some(why) => {
4541 crate::recovery::restore_repair(&wt, &pre)
4544 .await
4545 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4546 self.state.event(
4547 "graft",
4548 format!("rolled back: {why}; the winner is untouched"),
4549 );
4550 GraftOutcome::RolledBack(why)
4551 }
4552 };
4553 self.state.graft = Some(GraftRecord {
4554 pre,
4555 post,
4556 lines,
4557 limit: None,
4558 agent: Some(agent_id),
4559 outcome,
4560 requested,
4561 adoption,
4562 adoption_error,
4563 });
4564 self.state.save()?;
4565 Ok(())
4566 }
4567
4568 async fn fold_losers(&mut self) -> Result<()> {
4571 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4572 return Ok(());
4573 };
4574 if !self.state.candidate_preservation.is_empty() {
4575 bail!("candidate preservation incomplete; refusing cleanup");
4576 }
4577 let repo = self.state.repo.clone();
4578 let _coordination = self.coordinate(&repo).await?;
4579 let mut folded = Vec::new();
4580 for i in 0..self.state.candidates.len() {
4581 let c = &self.state.candidates[i];
4582 if c.label == winner || c.folded {
4583 continue;
4584 }
4585 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4586 remove_preserved_worktree(&repo, &wt).await?;
4587 if git::branch_exists(&repo, &branch).await?
4588 && !git::branch_delete(&repo, &branch).await?
4589 {
4590 bail!("candidate branch removal refused");
4591 }
4592 self.state.candidates[i].folded = true;
4593 self.state.save()?;
4594 folded.push(label.to_string());
4595 }
4596 let root = self.state.worktree_root();
4598 for j in 1..=self.roles.judges.len() {
4599 let wt = root.join(format!("judge-{j}"));
4600 if wt.exists() {
4601 remove_preserved_worktree(&repo, &wt).await?;
4602 }
4603 }
4604 if self.state.config.graph.advise {
4607 for k in 1..=self.state.config.graph.advisors {
4608 let wt = root.join(format!("advisor-{k}"));
4609 if wt.exists() {
4610 remove_preserved_worktree(&repo, &wt).await?;
4611 }
4612 }
4613 }
4614 if !folded.is_empty() {
4615 self.state
4616 .event("fold", format!("folded candidates {}", folded.join(", ")));
4617 self.state.save()?;
4618 }
4619 Ok(())
4620 }
4621
4622 async fn sync_to_base(&mut self) -> Result<()> {
4662 if self.state.status == RunStatus::AlreadyInBase {
4663 return Ok(());
4664 }
4665 let conflicted = self
4666 .state
4667 .base_sync
4668 .as_ref()
4669 .is_some_and(|s| s.conflict.is_some());
4670 let Some(winner) = self.state.winner().cloned() else {
4671 return Ok(());
4672 };
4673
4674 let repo = self.state.repo.clone();
4675 let remote = self.state.config.merge.remote.clone();
4676 let base_branch = self.state.base_branch.clone();
4677 let tracking = format!("{remote}/{base_branch}");
4678
4679 git::fetch(&repo, &remote, &base_branch).await.ok();
4680 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4685 return Ok(());
4686 };
4687
4688 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4689 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4690 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4691
4692 if (behind > 0 || conflicted || head == tip)
4701 && self
4702 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4703 .await?
4704 {
4705 return Ok(());
4706 }
4707 if conflicted {
4708 return Ok(());
4709 }
4710
4711 if behind == 0 {
4712 if let Some(from) = self
4721 .state
4722 .rebase_fixes
4723 .iter()
4724 .rev()
4725 .find_map(|r| r.from.clone())
4726 && from != head
4727 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4728 .await
4729 .is_ok_and(|o| o.ok())
4730 {
4731 git::sync_to_head(&winner.worktree).await?;
4732 }
4733 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4739 if let Some(why) = &conflict {
4740 self.state.status = RunStatus::Blocked;
4741 self.state.event("land", why.clone());
4742 }
4743 self.state.base_sync = Some(BaseSync {
4744 tip,
4745 behind: 0,
4746 attempts,
4747 conflict,
4748 already_in: None,
4749 });
4750 self.state.save()?;
4751 return Ok(());
4752 }
4753
4754 if attempts >= BASE_SYNC_ROUNDS {
4755 let why = format!(
4756 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4757 rebase(s); rebasing again would only race it",
4758 winner.branch
4759 );
4760 self.state.status = RunStatus::Blocked;
4761 self.state.base_sync = Some(BaseSync {
4762 tip,
4763 behind,
4764 attempts,
4765 conflict: Some(why.clone()),
4766 already_in: None,
4767 });
4768 self.state.event("land", why);
4769 self.state.save()?;
4770 return Ok(());
4771 }
4772
4773 self.state.event(
4774 "land",
4775 format!(
4776 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4777 winner.branch
4778 ),
4779 );
4780 self.state.save()?;
4781
4782 let branch_tracking = format!("{remote}/{}", winner.branch);
4788 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4789 let remote_tip = if self.state.task_pr_branch().is_none()
4790 && matches!(&fetched_branch, Ok(o) if o.ok())
4791 {
4792 git::rev_parse(&repo, &branch_tracking).await.ok()
4793 } else {
4794 None
4795 };
4796 if let Some(theirs) = &remote_tip
4800 && !git::is_ancestor(&repo, theirs, &head).await
4801 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4802 .await
4803 .is_ok_and(|missing| missing.is_empty())
4804 {
4805 let why = format!(
4806 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4807 them",
4808 short(theirs),
4809 winner.branch
4810 );
4811 self.state.status = RunStatus::Blocked;
4812 self.state.base_sync = Some(BaseSync {
4813 tip,
4814 behind,
4815 attempts,
4816 conflict: Some(why.clone()),
4817 already_in: None,
4818 });
4819 self.state.event("land", why);
4820 self.state.save()?;
4821 return Ok(());
4822 }
4823
4824 let scratch = self.state.dir().join("base-sync");
4825 let rebased = match crate::rebase::rebase_with_fixer(
4826 &mut self.state,
4827 &scratch,
4828 &winner.branch,
4829 &tracking,
4830 )
4831 .await
4832 {
4833 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4834 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4835 Err(e) => Err(e),
4836 };
4837 let attempts = attempts + 1;
4838 match rebased {
4839 Ok(None) => {
4840 git::sync_to_head(&winner.worktree).await?;
4844 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4845 let mut conflict = None;
4846 if let Some(pinned) = &remote_tip {
4847 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4848 match pushed {
4849 Ok(o) if o.ok() => self.state.event(
4850 "land",
4851 format!("pushed rebased {} to {remote}", winner.branch),
4852 ),
4853 Ok(o) => {
4854 conflict = Some(format!(
4855 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4856 winner.branch,
4857 short(pinned),
4858 o.stderr.chars().take(600).collect::<String>()
4859 ));
4860 }
4861 Err(e) => {
4862 conflict = Some(format!(
4863 "rebased {} locally but could not push it: {e:#}",
4864 winner.branch
4865 ));
4866 }
4867 }
4868 }
4869 if let Some(why) = &conflict {
4870 self.state.status = RunStatus::Blocked;
4871 self.state.event("land", why.clone());
4872 }
4873 self.state.base_sync = Some(BaseSync {
4874 tip: tip.clone(),
4875 behind: 0,
4876 attempts,
4877 conflict,
4878 already_in: None,
4879 });
4880 self.state
4881 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4882 }
4883 Ok(Some(conflict)) => {
4884 let why = format!(
4885 "{} conflicts with {tracking} and did not rebase: {}",
4886 winner.branch,
4887 conflict.chars().take(600).collect::<String>()
4888 );
4889 self.state.status = RunStatus::Blocked;
4890 self.state.base_sync = Some(BaseSync {
4891 tip,
4892 behind,
4893 attempts,
4894 conflict: Some(why.clone()),
4895 already_in: None,
4896 });
4897 self.state.event("land", why);
4898 }
4899 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4903 self.state.save()?;
4904 return Ok(());
4905 }
4906 Err(e) => {
4907 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4908 self.state.status = RunStatus::Blocked;
4909 self.state.base_sync = Some(BaseSync {
4910 tip,
4911 behind,
4912 attempts,
4913 conflict: Some(why.clone()),
4914 already_in: None,
4915 });
4916 self.state.event("land", why);
4917 }
4918 }
4919 self.state.save()?;
4920 Ok(())
4921 }
4922
4923 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4929 if self.state.task_pr_branch().is_some() {
4932 return None;
4933 }
4934 let repo = self.state.repo.clone();
4935 let remote = self.state.config.merge.remote.clone();
4936 let fetched = git::fetch(&repo, &remote, branch).await;
4937 if !matches!(&fetched, Ok(o) if o.ok()) {
4938 return None;
4939 }
4940 let branch_tracking = format!("{remote}/{branch}");
4941 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4942 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4943 return None;
4944 }
4945 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4946 .await
4947 .is_ok_and(|missing| missing.is_empty())
4948 {
4949 return Some(format!(
4950 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4951 them",
4952 short(&theirs)
4953 ));
4954 }
4955 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4956 Ok(o) if o.ok() => {
4957 self.state
4958 .event("land", format!("pushed rebased {branch} to {remote}"));
4959 None
4960 }
4961 Ok(o) => Some(format!(
4962 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4963 someone may have pushed): {}",
4964 short(&theirs),
4965 o.stderr.chars().take(600).collect::<String>()
4966 )),
4967 Err(e) => Some(format!(
4968 "{branch} is rebased locally but could not be pushed: {e:#}"
4969 )),
4970 }
4971 }
4972
4973 async fn settle_already_in(
4990 &mut self,
4991 branch: &str,
4992 tip: &str,
4993 head: &str,
4994 attempts: usize,
4995 behind: usize,
4996 ) -> Result<bool> {
4997 let repo = self.state.repo.clone();
4998 let remote = self.state.config.merge.remote.clone();
4999 let start = self.state.base_commit.clone();
5000 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5001 Ok(Some(e)) => e,
5002 Ok(None) => return Ok(false),
5003 Err(e) => {
5004 tracing::warn!("already-in-base check for {branch}: {e:#}");
5005 return Ok(false);
5006 }
5007 };
5008 let mut verified = vec![head.to_owned()];
5009 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5010 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5011 if matches!(&fetched, Ok(o) if o.ok())
5012 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5013 && theirs != head
5014 {
5015 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5016 Ok(Some(_)) => verified.push(theirs),
5017 _ => return Ok(false),
5018 }
5019 }
5020 let base_branch = self.state.base_branch.clone();
5021 let message = format!(
5022 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5023 land",
5024 evidence.names(),
5025 evidence.proof.as_str()
5026 );
5027 let closed =
5028 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5029 .await;
5030 match closed {
5031 Ok(Ok(url)) => self
5032 .state
5033 .event("land", format!("closed {url}: superseded on {base_branch}")),
5034 Ok(Err(why)) => self
5035 .state
5036 .event("land", format!("did not close a pull request: {why}")),
5037 Err(e) => {
5038 let why = format!(
5039 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5040 not be closed ({e:#}); resume to retry"
5041 );
5042 self.state.status = RunStatus::Blocked;
5043 self.state.base_sync = Some(BaseSync {
5044 tip: tip.to_owned(),
5045 behind,
5046 attempts,
5047 conflict: Some(why.clone()),
5048 already_in: None,
5049 });
5050 self.state.event("land", why);
5051 self.state.save()?;
5052 return Ok(true);
5053 }
5054 }
5055 self.state.status = RunStatus::AlreadyInBase;
5056 self.state.base_sync = Some(BaseSync {
5057 tip: tip.to_owned(),
5058 behind,
5059 attempts,
5060 conflict: None,
5061 already_in: Some(evidence),
5062 });
5063 self.state.event("land", message);
5064 self.state.save()?;
5065 self.settle_questions();
5066 Ok(true)
5067 }
5068
5069 fn landing_base(&self) -> String {
5079 self.state
5080 .base_sync
5081 .as_ref()
5082 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5083 }
5084
5085 pub async fn fix_selected(
5118 &mut self,
5119 ids: &[String],
5120 reason: &str,
5121 allow_stale: bool,
5122 ) -> Result<()> {
5123 let reason = reason.trim();
5124 if reason.is_empty() {
5125 bail!("a fix request needs a reason — that is the operator's own record of why");
5126 }
5127 if ids.is_empty() {
5128 bail!("no finding id given");
5129 }
5130 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5131 bail!(
5132 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5133 has already concluded — can be given a targeted fix. A run still \
5134 in progress should simply be resumed; a `merged` run's branch has \
5135 already landed, so its answer is a fresh `magi review <branch>`, \
5136 not reopening this run's own record",
5137 self.state.id,
5138 self.state.status.as_str()
5139 );
5140 }
5141 let Some(winner) = self.state.winner().cloned() else {
5142 bail!("run {} has no winning candidate to fix", self.state.id);
5143 };
5144 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5145 bail!(
5146 "branch `{}` no longer exists; this run cannot be extended",
5147 winner.branch
5148 );
5149 }
5150 let home = crate::run::home();
5151 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5152 bail!(
5153 "run {} is currently being worked on by another magi process",
5154 self.state.id
5155 );
5156 }
5157 let _claim = FixClaim::acquire(&self.state.dir())?;
5163
5164 let mut seen = BTreeSet::new();
5168 let mut findings = Vec::new();
5169 let mut missing = Vec::new();
5170 for id in ids {
5171 if !seen.insert(id.clone()) {
5172 continue;
5173 }
5174 match self.state.finding(id) {
5175 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5176 id: f.id.clone(),
5177 severity: f.severity,
5178 reviewer_vote: rec.vote,
5179 round: round.round,
5180 round_head: round.head.clone(),
5181 reviewer: rec.reviewer,
5182 agent: rec.agent.clone(),
5183 file: f.file.clone(),
5184 line: f.line,
5185 title: f.title.clone(),
5186 detail: f.detail.clone(),
5187 outcome: OperatorFixOutcome::Pending,
5188 }),
5189 None => missing.push(id.clone()),
5190 }
5191 }
5192 if !missing.is_empty() {
5193 bail!(
5194 "unknown finding id(s): {}; nothing was changed",
5195 missing.join(", ")
5196 );
5197 }
5198
5199 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5200 let stale_details: Vec<(String, String)> = findings
5201 .iter()
5202 .filter(|f| f.round_head != head_at_request)
5203 .map(|f| (f.id.clone(), f.round_head.clone()))
5204 .collect();
5205 let stale = !stale_details.is_empty();
5206 if stale && !allow_stale {
5207 bail!(
5208 "the branch has moved since some finding(s) were raised — {} — now \
5209 at {}; pass --allow-stale to fix anyway, or re-run review first",
5210 stale_details
5211 .iter()
5212 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5213 .collect::<Vec<_>>()
5214 .join(", "),
5215 short(&head_at_request)
5216 );
5217 }
5218
5219 let request = OperatorFixRequest {
5220 requested_at: Timestamp::now(),
5221 reason: reason.to_owned(),
5222 findings,
5223 head_at_request: head_at_request.clone(),
5224 allow_stale,
5225 stale,
5226 fix: None,
5227 result_head: None,
5228 follow_up_review_run: None,
5229 };
5230 self.state.event(
5231 "fix",
5232 format!(
5233 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5234 request.findings.len(),
5235 request
5236 .findings
5237 .iter()
5238 .map(|f| f.id.as_str())
5239 .collect::<Vec<_>>()
5240 .join(", "),
5241 ),
5242 );
5243 self.state.operator_fixes.push(request);
5250 self.state.save()?;
5251 let request_index = self.state.operator_fixes.len() - 1;
5252
5253 let repo = self.state.repo.clone();
5262 let coordination = self.coordinate(&repo).await?;
5263 if winner.worktree.exists() {
5264 let dirty = git::git(
5267 &winner.worktree,
5268 &["status", "--porcelain", "--untracked-files=all"],
5269 )
5270 .await?;
5271 let only_withheld = dirty.lines().all(|l| {
5272 l.strip_prefix("?? ")
5273 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5274 });
5275 if !only_withheld {
5276 bail!(
5277 "`{}` has uncommitted changes; refusing to touch it — commit or \
5278 discard them first",
5279 winner.worktree.display()
5280 );
5281 }
5282 git::worktree_remove(&self.state.repo, &winner.worktree)
5283 .await
5284 .ok();
5285 }
5286 let fix_worktree = self.state.worktree_root().join("operator-fix");
5287 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5288 git::git(
5289 &self.state.repo,
5290 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5291 )
5292 .await
5293 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5294 if !git::is_clean(&fix_worktree).await? {
5295 git::worktree_remove(&self.state.repo, &fix_worktree)
5296 .await
5297 .ok();
5298 bail!(
5299 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5300 winner.branch
5301 );
5302 }
5303 let repo_for_setup = self.state.repo.clone();
5304 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5305 Ok(()) => {
5306 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5307 }
5308 Err(e) => Err(e),
5309 };
5310 if let Err(e) = setup {
5311 git::worktree_remove(&self.state.repo, &fix_worktree)
5312 .await
5313 .ok();
5314 return Err(e);
5315 }
5316
5317 let run_id = self.state.id.clone();
5318 let prompts = self.state.config.prompts.clone();
5319 let language = self.state.config.graph.language.clone();
5320 let sessions = self.state.config.graph.sessions;
5321 let artifacts = agent::artifacts_dir(&self.state.dir());
5322 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5323 .findings
5324 .iter()
5325 .map(|f| Finding {
5326 id: f.id.clone(),
5327 severity: f.severity,
5328 file: f.file.clone(),
5329 line: f.line,
5330 title: f.title.clone(),
5331 detail: f.detail.clone(),
5332 })
5333 .collect();
5334 let fix_prompt = prompt::operator_fix_in(
5335 &self.state.instruction,
5336 &finding_list,
5337 reason,
5338 &stale_details,
5339 &head_at_request,
5340 &language,
5341 &self.state.config.graph.github_language,
5342 );
5343 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5344 drop(coordination);
5345 let (job, seat, out) = self
5346 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5347 prompt: fix_prompt.clone(),
5348 spec,
5349 seat,
5350 cwd: fix_worktree.clone(),
5351 timeout,
5352 allow_write: true,
5353 sessions,
5354 artifacts: artifacts.clone(),
5355 stem: "operator-fix".to_owned(),
5356 handover: None,
5357 })
5358 .await;
5359 let agent_id = seat.agent.clone();
5360
5361 let mut fix = FixRecord {
5362 agent: agent_id,
5363 addressed: Vec::new(),
5364 rejected: Vec::new(),
5365 notes: String::new(),
5366 committed: false,
5367 failed: None,
5368 duration_ms: 0,
5369 continuation: None,
5370 };
5371 let mut final_seat = seat.clone();
5372 match out {
5373 AgentOutcome::Ok(o) => {
5374 fix.duration_ms = o.duration_ms;
5375 let parsed = verdict::extract_json::<FixReport>(&o.text);
5376 let incomplete_reason = match &parsed {
5377 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5378 "the reply parsed, but it reported a command whose own CLI \
5379 never confirmed an exit status"
5380 .to_owned(),
5381 ),
5382 Ok(_) => None,
5383 Err(e) => Some(e.to_string()),
5384 };
5385 match incomplete_reason {
5386 None => {
5387 let report = parsed.expect("checked Ok above");
5388 fix.addressed = report.addressed;
5389 fix.rejected = report.rejected;
5390 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5391 }
5392 Some(reason) => {
5393 let (resumed_seat, resolved, failure, cont) = self
5394 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5395 .await;
5396 fix.duration_ms += cont.cumulative_wait_ms;
5397 fix.continuation = Some(cont);
5398 final_seat = resumed_seat;
5399 match resolved {
5400 Some(report) => {
5401 fix.addressed = report.addressed;
5402 fix.rejected = report.rejected;
5403 fix.notes =
5404 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5405 }
5406 None => fix.failed = failure,
5407 }
5408 }
5409 }
5410 }
5411 AgentOutcome::Dropped(o) => {
5412 fix.duration_ms = o.duration_ms;
5413 let why = o
5414 .dropped
5415 .as_ref()
5416 .map(|d| d.why.as_str())
5417 .unwrap_or("the CLI ended the stream without delivering its answer");
5418 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5419 }
5420 AgentOutcome::Quota(o) => {
5421 self.state.quota.push(QuotaLoss {
5422 seat: final_seat.key.clone(),
5423 node: "fix".to_owned(),
5424 at: Timestamp::now(),
5425 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5426 });
5427 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5428 }
5429 AgentOutcome::Failed(e) => fix.failed = Some(e),
5430 AgentOutcome::ResourceBlocked(e) => {
5431 fix.failed = Some(format!("the fixer was not started: {e}"));
5432 }
5433 }
5434 if fix.continuation.is_none() {
5435 fix.continuation = Some(ContinuationRecord::not_needed());
5436 }
5437 self.state.seats.insert(final_seat.key.clone(), final_seat);
5438
5439 let rescue_message = format!(
5440 "magi: operator-selected fix ({}) (uncommitted work)",
5441 self.state.operator_fixes[request_index]
5442 .findings
5443 .iter()
5444 .map(|f| f.id.as_str())
5445 .collect::<Vec<_>>()
5446 .join(", ")
5447 );
5448 let repo = self.state.repo.clone();
5449 let coordination = self.coordinate(&repo).await?;
5450 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5451 self.state.note_withheld("fix", &r.withheld);
5452 }
5453 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5454 fix.committed = after != head_at_request;
5455 git::worktree_remove(&self.state.repo, &fix_worktree)
5456 .await
5457 .ok();
5458 drop(coordination);
5459
5460 self.state.event(
5461 "fix",
5462 match &fix.failed {
5463 Some(reason) => format!(
5464 "operator fix: adoption report was lost ({reason}); {}",
5465 if fix.committed {
5466 "committed"
5467 } else {
5468 "NO new commit"
5469 }
5470 ),
5471 None => format!(
5472 "operator fix: {} addressed, {} rejected, {}",
5473 fix.addressed.len(),
5474 fix.rejected.len(),
5475 if fix.committed {
5476 "committed"
5477 } else {
5478 "NO new commit"
5479 }
5480 ),
5481 },
5482 );
5483
5484 for f in &mut self.state.operator_fixes[request_index].findings {
5491 f.outcome = if fix.failed.is_some() {
5492 OperatorFixOutcome::Unreported
5493 } else if fix.addressed.contains(&f.id) {
5494 OperatorFixOutcome::Addressed
5495 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5496 OperatorFixOutcome::Rejected { why: r.why.clone() }
5497 } else {
5498 OperatorFixOutcome::Unreported
5499 };
5500 }
5501
5502 let committed = fix.committed;
5503 if committed {
5504 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5505 }
5506 self.state.operator_fixes[request_index].fix = Some(fix);
5507 self.state.save()?;
5510
5511 if committed {
5512 self.state.event(
5513 "fix",
5514 format!(
5515 "operator fix committed {}; opening a follow-up review-only run",
5516 short(&after)
5517 ),
5518 );
5519 let origin =
5522 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5523 match Self::review(
5524 &self.state.repo,
5525 &winner.branch,
5526 self.state.config.clone(),
5527 origin,
5528 )
5529 .await
5530 {
5531 Ok(mut follow_up) => {
5532 follow_up.state.event(
5533 "start",
5534 format!(
5535 "requested by an operator fix on run {} for finding(s) {}",
5536 self.state.id,
5537 self.state.operator_fixes[request_index]
5538 .findings
5539 .iter()
5540 .map(|f| f.id.as_str())
5541 .collect::<Vec<_>>()
5542 .join(", "),
5543 ),
5544 );
5545 follow_up.state.save()?;
5546 let follow_up_id = follow_up.state.id.clone();
5547 let adopted = match crate::direct::adopt(
5550 &follow_up.state,
5551 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5552 ) {
5553 Ok(a) => a,
5554 Err(e) => {
5555 self.state.event(
5558 "fix",
5559 format!(
5560 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5561 ),
5562 );
5563 self.state.save()?;
5564 return Ok(());
5565 }
5566 };
5567 let executed = follow_up.execute().await;
5568 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5569 if let Some(a) = adopted {
5570 a.finish(&follow_up.state, executed);
5571 }
5572 if let Some(e) = failure {
5573 self.state.event(
5574 "fix",
5575 format!(
5576 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5577 ),
5578 );
5579 }
5580 self.state.operator_fixes[request_index].follow_up_review_run =
5581 Some(follow_up_id);
5582 }
5583 Err(e) => {
5584 self.state.event(
5585 "fix",
5586 format!("committed the fix but could not open a follow-up review: {e:#}"),
5587 );
5588 }
5589 }
5590 self.state.save()?;
5591 }
5592
5593 Ok(())
5594 }
5595
5596 async fn ask_fixer(
5616 &mut self,
5617 winner: &Candidate,
5618 node: &'static str,
5619 round: Option<usize>,
5620 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5621 ) -> (SeatJob, SeatState, AgentOutcome) {
5622 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5623 self.ask_fixer_among(attempts, node, round, None, build)
5624 .await
5625 }
5626
5627 async fn ask_fixer_among(
5632 &mut self,
5633 attempts: Vec<(AgentSpec, String)>,
5634 node: &'static str,
5635 round: Option<usize>,
5636 restore: Option<(&Path, &str)>,
5637 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5638 ) -> (SeatJob, SeatState, AgentOutcome) {
5639 let run_id = self.state.id.clone();
5640 let prompts = self.state.config.prompts.clone();
5641 let cache = self.state.config.cache_dir();
5642 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5643 let mut last = None;
5644 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5645 let seat = self.seat(&key, &spec.id);
5646 let mut job = build(spec.clone(), seat);
5647 if i > 0 {
5648 job.stem = format!("{}-{}", job.stem, spec.id);
5649 }
5650 let ctx = WaveCtx {
5651 carry_seats: false,
5652 pauses: Some((&self.pause, &self.interrupt)),
5653 run: &run_id,
5654 node,
5655 prompts: &prompts,
5656 cache: cache.as_deref(),
5657 round,
5658 };
5659 let (seat, out) =
5660 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5661 let advances = match &out {
5664 AgentOutcome::Ok(o) => agent::output_advances(o),
5665 AgentOutcome::ResourceBlocked(_) => false,
5666 _ => true,
5667 };
5668 if let Some(next) = ids.get(i + 1)
5669 && advances
5670 {
5671 self.state.seats.insert(seat.key.clone(), seat.clone());
5672 let class =
5673 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5674 let reason = match &out {
5675 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5676 other => fail_reason(other),
5677 };
5678 if let Some((wt, rev)) = restore
5682 && let Err(e) = self.graft_restore(wt, rev).await
5683 {
5684 let why =
5685 format!("could not restore the worktree before the next fixer: {e:#}");
5686 tracing::warn!("{why}");
5687 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5688 AgentOutcome::ResourceBlocked(why)
5689 } else {
5690 AgentOutcome::Failed(why)
5691 };
5692 last = Some((job, seat, out));
5693 break;
5694 }
5695 record_handover(
5696 &mut self.state,
5697 node,
5698 &seat.key,
5699 &spec.id,
5700 next,
5701 &class,
5702 &reason,
5703 );
5704 continue;
5705 }
5706 last = Some((job, seat, out));
5707 break;
5708 }
5709 last.expect("the fixer chain always has an entry")
5710 }
5711
5712 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5723 use crate::recovery::{Mode, Repair};
5724 if !self.state.reviews.is_empty() {
5725 return Ok(());
5726 }
5727 let Some(mut rec) = self.state.recovery.clone() else {
5728 return Ok(());
5729 };
5730 if rec.mode != Mode::VerifyRepair {
5731 return Ok(());
5732 }
5733 let wt = winner.worktree.clone();
5734 let mut head = git::rev_parse(&wt, "HEAD").await?;
5735 match rec.repair.clone() {
5736 Repair::Done { .. } => return Ok(()),
5737 Repair::WaitingForRestore { before } => {
5738 self.graft_restore(&wt, &before).await?;
5739 head = before;
5740 rec.repair = Repair::NotStarted;
5741 self.state.recovery = Some(rec.clone());
5742 self.state.save()?;
5743 }
5744 Repair::Started { before } => {
5745 if let Err(e) = self.graft_restore(&wt, &before).await {
5749 if e.is::<crate::repo_lock::ResourceBlocked>() {
5750 return Err(e);
5751 }
5752 return self
5753 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5754 .await;
5755 }
5756 rec.repair = Repair::Done {
5757 before: before.clone(),
5758 after: before,
5759 committed: false,
5760 failed: Some("interrupted; rolled back".to_owned()),
5761 };
5762 self.state
5763 .event("recovery", "the repair was interrupted and rolled back");
5764 self.state.recovery = Some(rec);
5765 self.state.save()?;
5766 return Ok(());
5767 }
5768 Repair::NotStarted => {}
5769 }
5770 if crate::recovery::recheck(&mut rec, &head) {
5771 self.state.event("recovery", rec.summary());
5772 self.state.recovery = Some(rec);
5773 self.state.save()?;
5774 return Ok(());
5775 }
5776 rec.repair = Repair::Started {
5777 before: head.clone(),
5778 };
5779 self.state.recovery = Some(rec.clone());
5780 self.state.save()?;
5781
5782 let language = self.state.config.graph.language.clone();
5783 let sessions = self.state.config.graph.sessions;
5784 let artifacts = agent::artifacts_dir(&self.state.dir());
5785 let fix_prompt = prompt::recovery_fix(
5786 &self.state.instruction,
5787 &rec,
5788 self.state.config.graph.review_rounds,
5789 &language,
5790 );
5791 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5792 self.state.event(
5793 "recovery",
5794 format!(
5795 "repair: the fixer starts from the failed verification on {}",
5796 short(&head)
5797 ),
5798 );
5799 let (_job, seat, out) = self
5800 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5801 prompt: fix_prompt.clone(),
5802 spec,
5803 seat,
5804 cwd: wt.clone(),
5805 timeout,
5806 allow_write: true,
5807 sessions,
5808 artifacts: artifacts.clone(),
5809 stem: "fix-recovery".to_owned(),
5810 handover: None,
5811 })
5812 .await;
5813 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5814 rec.repair = Repair::WaitingForRestore { before: head };
5815 self.state.recovery = Some(rec);
5816 return self.stop_for_resource();
5817 }
5818 let failed = match &out {
5819 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5820 AgentOutcome::Quota(o) => {
5821 self.state.quota.push(QuotaLoss {
5822 seat: seat.key.clone(),
5823 node: "fix".to_owned(),
5824 at: Timestamp::now(),
5825 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5826 });
5827 Some("rate limited (quota); the fixer could not run".to_owned())
5828 }
5829 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5830 AgentOutcome::Failed(e) => Some(e.clone()),
5831 };
5832 self.state.seats.insert(seat.key.clone(), seat);
5833 let coordination = self.coordinate(&wt).await?;
5834 let failed = match failed {
5835 Some(why) => {
5837 if let Err(e) = git::restore_to(&wt, &head).await {
5838 return self
5839 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5840 .await;
5841 }
5842 Some(why)
5843 }
5844 None => {
5845 if let Ok(r) = git::rescue_commit(
5846 &wt,
5847 "magi: recovery repair of the failed verification (uncommitted work)",
5848 )
5849 .await
5850 {
5851 self.state.note_withheld("fix", &r.withheld);
5852 }
5853 None
5854 }
5855 };
5856 let after = git::rev_parse(&wt, "HEAD").await?;
5857 drop(coordination);
5858 self.state.event(
5859 "recovery",
5860 match &failed {
5861 Some(why) => format!("repair: nothing kept ({why})"),
5862 None if after != head => {
5863 format!("repair: committed {}", short(&after))
5864 }
5865 None => "repair: the fixer made no commit".to_owned(),
5866 },
5867 );
5868 rec.repair = Repair::Done {
5869 before: head.clone(),
5870 committed: after != head,
5871 after,
5872 failed,
5873 };
5874 self.state.recovery = Some(rec);
5875 self.state.save()?;
5876 Ok(())
5877 }
5878
5879 async fn stop_recovery(
5882 &mut self,
5883 mut rec: crate::recovery::RecoveryRecord,
5884 head: &str,
5885 why: String,
5886 ) -> Result<()> {
5887 rec.repair = crate::recovery::Repair::Started {
5889 before: head.to_owned(),
5890 };
5891 self.state.recovery = Some(rec);
5892 self.state
5893 .event("recovery", format!("repair could not be undone: {why}"));
5894 self.state.status = RunStatus::Blocked;
5895 self.state.save()?;
5896 bail!("recovery repair could not be rolled back: {why}")
5897 }
5898
5899 async fn review_loop(&mut self) -> Result<()> {
5900 loop {
5901 if self
5902 .state
5903 .base_sync
5904 .as_ref()
5905 .is_some_and(|s| s.conflict.is_some())
5906 {
5907 return Ok(());
5908 }
5909 if self.catchup_repairable() {
5910 let record = self.state.reviews.last().expect("repairable round").clone();
5911 let winner = self
5912 .state
5913 .winner()
5914 .cloned()
5915 .context("catch-up repair has no winner")?;
5916 if !self.fix_review_round(&winner, record).await? {
5917 return Ok(());
5918 }
5919 }
5920 self.review_loop_pass().await?;
5921 if !self.catchup_repairable() {
5922 return Ok(());
5923 }
5924 }
5925 }
5926
5927 fn catchup_repairable(&self) -> bool {
5930 self.state.reviews.len() < self.state.config.graph.review_rounds
5931 && self.state.reviews.last().is_some_and(|r| {
5932 !r.clean
5933 && r.blocking == 0
5934 && r.fix.is_none()
5935 && r.e2e_status() == E2eStatus::Failed
5936 })
5937 }
5938
5939 async fn review_loop_pass(&mut self) -> Result<()> {
5940 if self
5945 .state
5946 .base_sync
5947 .as_ref()
5948 .is_some_and(|s| s.conflict.is_some())
5949 {
5950 return Ok(());
5951 }
5952 let run_id = self.state.id.clone();
5957 let prompts = self.state.config.prompts.clone();
5958 let Some(winner) = self.state.winner().cloned() else {
5959 return Ok(());
5960 };
5961 let max_rounds = self.state.config.graph.review_rounds;
5962 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5972 if status == RunStatus::Gating {
5975 self.record_contested_handoff();
5976 }
5977 self.state.status = status;
5978 self.state.save()?;
5979 return Ok(());
5980 }
5981 self.state.status = RunStatus::Reviewing;
5982 if self.state.reviews.last().is_some_and(|r| {
5992 r.e2e_status() == E2eStatus::ResourceBlocked
5993 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5994 }) {
5995 let shell = self.state.config.shell();
5996 return self
5997 .stop_unresolved_review(
5998 "the last round's own verification never resolved",
5999 &shell,
6000 &winner.worktree,
6001 )
6002 .await;
6003 }
6004
6005 self.recovery_repair(&winner).await?;
6006 if self.resource_halt()? {
6007 return Ok(());
6008 }
6009
6010 let repo = self.state.repo.clone();
6011 let root = self.state.worktree_root();
6012 let language = self.state.config.graph.language.clone();
6013 let sessions = self.state.config.graph.sessions;
6014 let artifacts = self.merge_repair_artifacts();
6015 let base = self.landing_base();
6016 let base_short = short(&base);
6017 let reviewers = self.roles.reviewers.clone();
6018 let shell = self.state.config.shell();
6019
6020 for round in (self.state.reviews.len() + 1)..=max_rounds {
6021 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6022 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6023 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6024 let prev_verification = self
6033 .state
6034 .reviews
6035 .last()
6036 .and_then(|r| r.verification_summary(&head));
6037 let prior_section = match (&self.state.recovery, round) {
6040 (Some(rec), 1) if self.state.reviews.is_empty() => {
6041 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6042 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6043 Some(crate::recovery::head_of(
6044 &git::diff(&winner.worktree, reviewed, "HEAD")
6045 .await
6046 .unwrap_or_default(),
6047 crate::recovery::SINCE_DIFF_MAX,
6048 ))
6049 }
6050 _ => None,
6051 };
6052 Some(prompt::recovery_review(rec, since.as_deref()))
6053 }
6054 _ => None,
6055 };
6056
6057 let mut jobs = Vec::new();
6061 let coordination = self.coordinate(&repo).await?;
6062 for (r, spec) in reviewers.iter().cloned().enumerate() {
6063 let wt = root.join(format!("review-{}", r + 1));
6064 if wt.exists() {
6067 git::reset_detached(&wt, &head).await?;
6068 } else {
6069 git::worktree_add_detached(&repo, &wt, &head).await?;
6070 }
6071 self.setup_seat_worktree(&repo, &wt, false).await?;
6072 let seat_key = format!("review-{}", r + 1);
6073 let spec = pick_start_spec(
6076 &self.roles.reviewer_roster,
6077 spec,
6078 self.state.seat_history.get(&seat_key),
6079 );
6080 let seat = self.seat(&seat_key, &spec.id);
6081 jobs.push(SeatJob {
6082 prompt: prompt::review(&prompt::ReviewCtx {
6083 instruction: &self.state.instruction,
6084 branch: &winner.branch,
6085 base_short: &base_short,
6086 stat: &stat,
6087 patch: &patch,
6088 verification: prev_verification.as_ref(),
6089 prior: prior_section.as_deref(),
6090 reviewers: reviewers.len(),
6091 round,
6092 rounds: max_rounds,
6093 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6096 lens: Lens::for_seat(r),
6097 language: &language,
6098 github_language: &self.state.config.graph.github_language,
6099 }),
6100 spec,
6101 seat,
6102 cwd: wt,
6103 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6104 allow_write: false,
6105 sessions,
6106 artifacts: artifacts.clone(),
6107 stem: format!("review-{round}-{}", r + 1),
6108 handover: None,
6109 });
6110 }
6111
6112 drop(coordination);
6113 self.state.event(
6114 "review",
6115 format!(
6116 "round {round}: {} reviewers on {}",
6117 jobs.len(),
6118 short(&head)
6119 ),
6120 );
6121 let mut quota_losses = Vec::new();
6122 let review_retries = self.state.config.graph.retries;
6123 let review_cache = self.state.config.cache_dir();
6124 let ctx = WaveCtx {
6125 carry_seats: true,
6126 pauses: Some((&self.pause, &self.interrupt)),
6127 run: &run_id,
6128 node: "review",
6129 prompts: &prompts,
6130 cache: review_cache.as_deref(),
6131 round: Some(round),
6132 };
6133 let results = ask_json_wave::<Review>(
6134 jobs,
6135 Arc::clone(&self.sem),
6136 review_retries,
6137 &self.roles.reviewer_roster,
6138 &ctx,
6139 &mut quota_losses,
6140 &mut self.state,
6141 &|_: &Review| Ok(()),
6142 )
6143 .await;
6144 if self.state.resource_block.is_some() {
6145 return self.stop_for_resource();
6146 }
6147 let round_quota_missing = quota_losses.len();
6151 self.state.quota.extend(quota_losses);
6152
6153 let mut records = Vec::new();
6154 let mut all_findings = Vec::new();
6155 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6156 let agent_id = seat.agent.clone();
6157 self.state.seats.insert(seat.key.clone(), seat);
6158 let mut record = ReviewRecord {
6159 reviewer: r + 1,
6160 agent: agent_id,
6161 summary: String::new(),
6162 findings: Vec::new(),
6163 vote: None,
6164 failed: None,
6165 duration_ms: 0,
6166 attempts,
6172 };
6173 match res {
6174 Ok((review, out)) => {
6175 record.summary =
6183 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6184 record.vote = Some(review.vote);
6185 record.duration_ms = out.duration_ms;
6186 for (n, mut f) in review.findings.into_iter().enumerate() {
6187 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6190 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6191 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6192 f.file = f
6198 .file
6199 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6200 all_findings.push(f.clone());
6201 record.findings.push(f);
6202 }
6203 self.state.event(
6204 "review",
6205 format!(
6206 "round {round}: reviewer {} voted {} with {} finding(s)",
6207 r + 1,
6208 review.vote.label(),
6209 record.findings.len()
6210 ),
6211 );
6212 }
6213 Err(e) => {
6214 record.failed = Some(e.to_string());
6215 self.state.event(
6216 "review",
6217 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6218 );
6219 }
6220 }
6221 records.push(record);
6222 }
6223
6224 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6231 let vote_split =
6232 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6233 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6234 if vote_split {
6235 self.state.event(
6236 "review",
6237 format!(
6238 "round {round}: votes split ({}) — one round of reconsideration",
6239 initial_votes
6240 .iter()
6241 .map(|v| v.label())
6242 .collect::<Vec<_>>()
6243 .join(", ")
6244 ),
6245 );
6246 let panel: Vec<ReviewSeatReport<'_>> = records
6249 .iter()
6250 .filter_map(|r| {
6251 r.vote.map(|vote| ReviewSeatReport {
6252 reviewer: r.reviewer,
6253 vote,
6254 summary: &r.summary,
6255 findings: &r.findings,
6256 })
6257 })
6258 .collect();
6259
6260 let mut jobs = Vec::new();
6261 let mut seats_at = Vec::new();
6262 for (r, spec) in reviewers.iter().cloned().enumerate() {
6263 if records[r].vote.is_none() {
6267 continue;
6268 }
6269 let wt = root.join(format!("review-{}", r + 1));
6270 let seat_key = format!("review-{}", r + 1);
6271 let spec = self.occupant(&seat_key, spec);
6272 let seat = self.seat(&seat_key, &spec.id);
6273 let build = |with_patch: bool| {
6281 prompt::review_reconsider(&ReviewReconsiderCtx {
6282 instruction: &self.state.instruction,
6283 reviewer: r + 1,
6284 lens: Lens::for_seat(r),
6285 panel: &panel,
6286 patch: with_patch.then_some(ReviewPatch {
6287 branch: &winner.branch,
6288 base_short: &base_short,
6289 stat: &stat,
6290 patch: &patch,
6291 }),
6292 round,
6293 rounds: max_rounds,
6294 language: &language,
6295 })
6296 };
6297 let full = build(true);
6298 let prompt = if has_context(&spec, &seat, sessions) {
6299 build(false)
6300 } else {
6301 full.clone()
6302 };
6303 jobs.push(SeatJob {
6304 prompt,
6305 spec,
6306 seat,
6307 cwd: wt,
6308 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6309 allow_write: false,
6310 sessions,
6311 artifacts: artifacts.clone(),
6312 stem: format!("review-{round}-reconsider-{}", r + 1),
6313 handover: Some(full),
6314 });
6315 seats_at.push(r);
6316 }
6317
6318 let mut recon_quota_losses = Vec::new();
6319 let recon_cache = self.state.config.cache_dir();
6320 let recon_ctx = WaveCtx {
6321 carry_seats: true,
6322 pauses: Some((&self.pause, &self.interrupt)),
6323 run: &run_id,
6324 node: "review",
6325 prompts: &prompts,
6326 cache: recon_cache.as_deref(),
6327 round: Some(round),
6328 };
6329 let recon_results = ask_json_wave::<ReviewRevote>(
6330 jobs,
6331 Arc::clone(&self.sem),
6332 review_retries,
6333 &self.roles.reviewer_roster,
6334 &recon_ctx,
6335 &mut recon_quota_losses,
6336 &mut self.state,
6337 &|_: &ReviewRevote| Ok(()),
6338 )
6339 .await;
6340 if self.state.resource_block.is_some() {
6341 return self.stop_for_resource();
6342 }
6343 self.state.quota.extend(recon_quota_losses);
6344
6345 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6346 let agent_id = seat.agent.clone();
6347 self.state.seats.insert(seat.key.clone(), seat);
6348 let mut rec = ReviewRevoteRecord {
6349 reviewer: r + 1,
6350 agent: agent_id,
6351 vote: None,
6352 reason: String::new(),
6353 failed: None,
6354 };
6355 match res {
6356 Ok((rv, _)) => {
6357 rec.vote = Some(rv.vote);
6358 rec.reason =
6359 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6360 self.state.event(
6361 "review",
6362 format!(
6363 "round {round}: reviewer {} revoted {}",
6364 r + 1,
6365 rv.vote.label()
6366 ),
6367 );
6368 }
6369 Err(e) => {
6370 rec.failed = Some(e.to_string());
6371 self.state.event(
6372 "review",
6373 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6374 );
6375 }
6376 }
6377 reconsideration.push(rec);
6378 }
6379 } else if initial_votes.len() > 1 {
6380 self.state.event(
6381 "review",
6382 format!(
6383 "round {round}: votes agreed ({}) — no reconsideration",
6384 initial_votes[0].label()
6385 ),
6386 );
6387 }
6388
6389 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6390 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6391 let defer_e2e =
6402 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6403 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6404 let reason =
6405 format!("{blocking} blocking finding(s) already required a fix this round");
6406 self.state.event(
6407 "verify",
6408 format!(
6409 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6410 {}); it will run once a round has none left",
6411 short(&head)
6412 ),
6413 );
6414 (Vec::new(), false, true, Some(reason))
6415 } else {
6416 let e2e_commands = self.state.config.verify.e2e.clone();
6417 let cache_dir = self.state.config.cache_dir();
6418 let context = format!("round {round}");
6419 let wait = self.repository_wait();
6420 let (e2e, verify_retried) = with_cache_lease(
6421 &mut self.state,
6422 cache_dir.as_deref(),
6423 "e2e",
6424 "e2e",
6425 &winner.worktree,
6426 &head,
6427 verify_timeout,
6428 &context,
6429 &wait,
6430 |state, budget| {
6431 let shell = shell.clone();
6432 let e2e_commands = e2e_commands.clone();
6433 let worktree = winner.worktree.clone();
6434 let context = context.clone();
6435 async move {
6436 run_e2e_with_retry(
6437 state,
6438 &shell,
6439 &e2e_commands,
6440 &worktree,
6441 budget,
6442 &context,
6443 )
6444 .await
6445 }
6446 },
6447 )
6448 .await?;
6449 (e2e, verify_retried, false, None)
6450 };
6451
6452 let expected = records.len();
6453 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6454 let incomplete = answered < expected;
6455 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6456 let policy = self.state.config.graph.incomplete_review;
6457 let clean = round_is_clean(
6458 blocking,
6459 e2e_ok,
6460 answered,
6461 expected,
6462 round_quota_missing,
6463 policy,
6464 );
6465
6466 let mut round_record = ReviewRound {
6467 round,
6468 head: head.clone(),
6469 verified_head: None,
6470 verified_at: None,
6471 reviews: records,
6472 e2e,
6473 verify_retried,
6474 e2e_deferred,
6475 e2e_defer_reason,
6476 fix: None,
6477 blocking,
6478 answered,
6479 expected,
6480 clean,
6481 progressed: false,
6482 vote_split,
6483 reconsideration,
6484 verdict: None,
6485 };
6486 round_record.verdict = ReviewVote::worst(
6490 round_record
6491 .final_votes()
6492 .into_iter()
6493 .map(|(_, _, vote)| vote),
6494 );
6495 if !matches!(
6506 round_record.e2e_status(),
6507 E2eStatus::Deferred | E2eStatus::NotConfigured
6508 ) {
6509 round_record.verified_head = Some(head.clone());
6510 round_record.verified_at = Some(Timestamp::now());
6511 }
6512
6513 if incomplete {
6514 let missing: Vec<String> = round_record
6515 .reviews
6516 .iter()
6517 .filter(|r| r.failed.is_some())
6518 .map(|r| format!("review-{}", r.reviewer))
6519 .collect();
6520 self.state.event(
6521 "review",
6522 format!(
6523 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6524 missing.join(", ")
6525 ),
6526 );
6527 }
6528
6529 if clean {
6530 self.state.event(
6531 "review",
6532 if incomplete && policy == IncompleteReviewPolicy::Warn {
6533 format!(
6534 "round {round}: clean (warn policy, incomplete panel) — no \
6535 blocking findings from the seats that answered, verification green"
6536 )
6537 } else if incomplete {
6538 format!(
6539 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6540 quorum) — no blocking findings from the seats that answered, \
6541 verification green",
6542 expected - answered
6543 )
6544 } else {
6545 format!("round {round}: clean — no blocking findings, verification green")
6546 },
6547 );
6548 self.state.reviews.push(round_record);
6549 self.state.status = RunStatus::Gating;
6550 self.state.save()?;
6551 return Ok(());
6552 }
6553
6554 if incomplete && blocking == 0 && e2e_ok {
6562 self.state.reviews.push(round_record);
6563 self.state.save()?;
6564 if round == max_rounds {
6565 self.state.status = RunStatus::Blocked;
6566 self.state.event(
6567 "review",
6568 format!(
6569 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6570 refusing to call it clean",
6571 expected - answered
6572 ),
6573 );
6574 return Ok(());
6575 }
6576 continue;
6577 }
6578
6579 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6591 self.state.reviews.push(round_record);
6592 return self
6593 .stop_unresolved_review(
6594 "the round's own verification could not run",
6595 &shell,
6596 &winner.worktree,
6597 )
6598 .await;
6599 }
6600
6601 if round == max_rounds {
6602 self.state.reviews.push(round_record);
6603 return self
6604 .stop_reviewing(
6605 &format!(
6606 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6607 ),
6608 &shell,
6609 &winner.worktree,
6610 )
6611 .await;
6612 }
6613
6614 if !self.fix_review_round(&winner, round_record).await? {
6615 return Ok(());
6616 }
6617 }
6618 Ok(())
6619 }
6620
6621 async fn fix_review_round(
6624 &mut self,
6625 winner: &Candidate,
6626 mut round_record: ReviewRound,
6627 ) -> Result<bool> {
6628 let round = round_record.round;
6629 let shell = self.state.config.shell();
6630 let step = self.run_fix_round(winner, &mut round_record).await?;
6631 if matches!(step, FixStep::Resource) {
6632 self.stop_for_resource()?;
6633 return Ok(false);
6634 }
6635 if self.state.reviews.len() < round {
6636 self.state.reviews.push(round_record);
6637 } else {
6638 self.state.reviews[round - 1] = round_record;
6639 }
6640 self.state.save()?;
6641
6642 if matches!(step, FixStep::Recorded { report_lost: true }) {
6655 self.stop_reviewing(
6656 "the fixer's adoption report never came back, even after resuming its \
6657 own seat; refusing to start another round against the same worktree \
6658 while that is unresolved",
6659 &shell,
6660 &winner.worktree,
6661 )
6662 .await?;
6663 return Ok(false);
6664 }
6665
6666 let streak = self
6667 .state
6668 .reviews
6669 .iter()
6670 .rev()
6671 .take_while(|r| !r.progressed)
6672 .count();
6673 if streak >= STAGNANT_LIMIT {
6674 self.stop_reviewing(
6675 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6676 &shell,
6677 &winner.worktree,
6678 )
6679 .await?;
6680 return Ok(false);
6681 }
6682 Ok(true)
6683 }
6684
6685 async fn run_fix_round(
6687 &mut self,
6688 winner: &Candidate,
6689 round_record: &mut ReviewRound,
6690 ) -> Result<FixStep> {
6691 let round = round_record.round;
6692 let max_rounds = self.state.config.graph.review_rounds;
6693 let run_id = self.state.id.clone();
6694 let prompts = self.state.config.prompts.clone();
6695 let language = self.state.config.graph.language.clone();
6696 let sessions = self.state.config.graph.sessions;
6697 let artifacts = self.merge_repair_artifacts();
6698 let base = self.landing_base();
6699 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6700 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6701 let this_round_verification = round_record.verification_summary(&head);
6702 let all_findings: Vec<_> = round_record
6703 .reviews
6704 .iter()
6705 .flat_map(|r| r.findings.iter().cloned())
6706 .collect();
6707 let blocking_findings: Vec<_> = all_findings
6710 .iter()
6711 .filter(|f| f.severity.blocks())
6712 .cloned()
6713 .collect();
6714 let fix_prompt = prompt::fix_in(
6715 &self.state.instruction,
6716 &blocking_findings,
6717 this_round_verification.as_ref(),
6718 round,
6719 max_rounds,
6720 &language,
6721 &self.state.config.graph.github_language,
6722 );
6723 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6724 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6725 let (job, seat, out) = self
6726 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6727 prompt: fix_prompt.clone(),
6728 spec,
6729 seat,
6730 cwd: winner.worktree.clone(),
6731 timeout,
6732 allow_write: true,
6733 sessions,
6734 artifacts: artifacts.clone(),
6735 stem: format!("fix-{round}"),
6736 handover: None,
6737 })
6738 .await;
6739 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6742 return Ok(FixStep::Resource);
6743 }
6744 let agent_id = seat.agent.clone();
6745
6746 let mut fix = FixRecord {
6747 agent: agent_id,
6748 addressed: Vec::new(),
6749 rejected: Vec::new(),
6750 notes: String::new(),
6751 committed: false,
6752 failed: None,
6753 duration_ms: 0,
6754 continuation: None,
6755 };
6756 let mut continuation = ContinuationRecord::not_needed();
6757 let mut final_seat = seat.clone();
6758 match out {
6759 AgentOutcome::Ok(o) => {
6760 fix.duration_ms = o.duration_ms;
6761 let parsed = verdict::extract_json::<FixReport>(&o.text);
6762 let incomplete_reason = match &parsed {
6769 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6770 "the reply parsed, but it reported a command whose own CLI never \
6771 confirmed an exit status"
6772 .to_owned(),
6773 ),
6774 Ok(_) => None,
6775 Err(e) => Some(e.to_string()),
6776 };
6777 match incomplete_reason {
6778 None => {
6779 let report = parsed.expect("checked Ok above");
6780 fix.addressed = report.addressed;
6781 fix.rejected = report.rejected;
6782 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6783 }
6784 Some(reason) => {
6785 let (resumed_seat, resolved, failure, cont) = self
6786 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6787 .await;
6788 fix.duration_ms += cont.cumulative_wait_ms;
6789 continuation = cont;
6790 final_seat = resumed_seat;
6791 match resolved {
6792 Some(report) => {
6793 fix.addressed = report.addressed;
6794 fix.rejected = report.rejected;
6795 fix.notes =
6796 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6797 }
6798 None => fix.failed = failure,
6799 }
6800 }
6801 }
6802 }
6803 AgentOutcome::Dropped(o) => {
6805 fix.duration_ms = o.duration_ms;
6806 let why = o
6807 .dropped
6808 .as_ref()
6809 .map(|d| d.why.as_str())
6810 .unwrap_or("the CLI ended the stream without delivering its answer");
6811 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6812 }
6813 AgentOutcome::Quota(o) => {
6814 self.state.quota.push(QuotaLoss {
6815 seat: final_seat.key.clone(),
6816 node: "fix".to_owned(),
6817 at: Timestamp::now(),
6818 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6819 });
6820 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6821 }
6822 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6823 }
6824 fix.continuation = Some(continuation);
6825 self.state.seats.insert(final_seat.key.clone(), final_seat);
6826 let coordination = self.coordinate(&winner.worktree).await?;
6827 if let Ok(r) = git::rescue_commit(
6828 &winner.worktree,
6829 &format!("magi: review round {round} fixes (uncommitted work)"),
6830 )
6831 .await
6832 {
6833 self.state.note_withheld("fix", &r.withheld);
6834 }
6835 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6836 fix.committed = after != before;
6837 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6845 drop(coordination);
6846 let progressed = diff_after != patch;
6847 let commit_note = if fix.committed {
6848 "committed"
6849 } else {
6850 "NO new commit"
6851 };
6852 let tree_note = if progressed {
6853 "changed vs base"
6854 } else {
6855 "unchanged vs base"
6856 };
6857 self.state.event(
6858 "fix",
6859 match &fix.failed {
6860 Some(reason) => {
6866 format!(
6867 "round {round}: fixer's adoption report was lost ({reason}); \
6868 {commit_note}, tree {tree_note}"
6869 )
6870 }
6871 None => format!(
6872 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6873 {tree_note}{}",
6874 fix.addressed.len(),
6875 fix.rejected.len(),
6876 if continuation.outcome == ContinuationOutcome::Resumed {
6877 format!(
6878 " (adoption report recovered after {} continuation(s))",
6879 continuation.attempts
6880 )
6881 } else {
6882 String::new()
6883 },
6884 ),
6885 },
6886 );
6887 round_record.fix = Some(fix);
6888 round_record.progressed = progressed;
6889 Ok(FixStep::Recorded {
6890 report_lost: matches!(
6891 continuation.outcome,
6892 ContinuationOutcome::Exhausted
6893 | ContinuationOutcome::QuotaLost
6894 | ContinuationOutcome::NoSession
6895 ),
6896 })
6897 }
6898
6899 fn record_contested_handoff(&mut self) {
6933 if self.state.contested_handoff.is_some() {
6934 return;
6935 }
6936 let Some(contested) = self
6937 .state
6938 .reviews
6939 .last()
6940 .and_then(ReviewRound::contested_handoff)
6941 else {
6942 return;
6943 };
6944 self.state.event(
6945 "review",
6946 format!(
6947 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6948 wait for the owner's approval",
6949 contested.findings.len(),
6950 contested.rejecters.len()
6951 ),
6952 );
6953 self.state.contested_handoff = Some(contested);
6954 }
6955
6956 async fn stop_unresolved_review(
6957 &mut self,
6958 why: &str,
6959 shell: &[String],
6960 worktree: &Path,
6961 ) -> Result<()> {
6962 self.stop_reviewing_inner(why, shell, worktree, true).await
6963 }
6964
6965 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6966 self.stop_reviewing_inner(why, shell, worktree, false).await
6967 }
6968
6969 async fn stop_reviewing_inner(
6970 &mut self,
6971 why: &str,
6972 shell: &[String],
6973 worktree: &Path,
6974 repairable: bool,
6975 ) -> Result<()> {
6976 let round_idx = self.state.reviews.len() - 1;
6977 let needs_catchup_run = matches!(
6985 self.state.reviews[round_idx].e2e_status(),
6986 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6987 );
6988 if needs_catchup_run {
6989 let round = self.state.reviews[round_idx].round;
6990 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6991 let commands = self.state.config.verify.e2e.clone();
6992 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6993 let cache_dir = self.state.config.cache_dir();
6994 let context = format!(
6995 "round {round}: verification unresolved, catching up before the final decision"
6996 );
6997 let wait = self.repository_wait();
6998 let (outcomes, verify_retried) = with_cache_lease(
6999 &mut self.state,
7000 cache_dir.as_deref(),
7001 "e2e",
7002 "e2e",
7003 worktree,
7004 &attempted_head,
7005 timeout,
7006 &context,
7007 &wait,
7008 |state, budget| {
7009 let shell = shell.to_vec();
7010 let commands = commands.clone();
7011 let context = context.clone();
7012 async move {
7013 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7014 .await
7015 }
7016 },
7017 )
7018 .await?;
7019 let last = &mut self.state.reviews[round_idx];
7020 last.e2e = outcomes;
7021 last.verify_retried = verify_retried;
7022 last.verified_head = Some(attempted_head);
7029 last.verified_at = Some(Timestamp::now());
7030 if verify_inconclusive(&last.e2e) {
7031 self.state.save()?;
7038 return Ok(());
7039 }
7040 last.e2e_deferred = false;
7041 }
7042 let last = &self.state.reviews[round_idx];
7043 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7044
7045 match last.e2e_status() {
7046 E2eStatus::Failed => {
7047 let red: Vec<String> = last
7048 .e2e
7049 .iter()
7050 .filter(|o| !o.ok())
7051 .map(|o| {
7052 format!(
7053 "`{}` -> {:?}\n{}",
7054 o.command,
7055 o.code,
7056 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7057 )
7058 })
7059 .collect();
7060 let can_repair = last.blocking == 0 && last.fix.is_none();
7061 self.state.reviews[round_idx].clean = false;
7062 self.state
7063 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7064 self.state.status = RunStatus::Blocked;
7065 if repairable
7066 && can_repair
7067 && self.state.reviews.len() < self.state.config.graph.review_rounds
7068 {
7069 self.state.status = RunStatus::Reviewing;
7070 }
7071 }
7072 E2eStatus::ResourceBlocked => {
7077 self.state.event(
7078 "review",
7079 format!(
7080 "{why}; e2e could not run (resource unavailable); not \
7081 deciding yet"
7082 ),
7083 );
7084 }
7085 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7086 self.state.event(
7087 "review",
7088 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7089 );
7090 self.record_contested_handoff();
7091 self.state.status = RunStatus::Gating;
7092 }
7093 }
7094 self.state.save()?;
7095 Ok(())
7096 }
7097
7098 async fn gate(&mut self) -> Result<()> {
7101 if self.state.status == RunStatus::Failed
7113 || self
7114 .state
7115 .base_sync
7116 .as_ref()
7117 .is_some_and(|s| s.conflict.is_some())
7118 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7119 != Some(RunStatus::Gating)
7120 {
7121 return Ok(());
7122 }
7123 if self.state.gate_ran {
7124 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7135 self.state.status = RunStatus::Blocked;
7136 self.state.save()?;
7137 }
7138 return Ok(());
7139 }
7140 let Some(winner) = self.state.winner().cloned() else {
7141 return Ok(());
7142 };
7143 self.state.status = RunStatus::Gating;
7144 let mut outcomes = self.run_gate(&winner).await?;
7145 loop {
7146 if verify_inconclusive(&outcomes) {
7157 self.state.save()?;
7158 return Ok(());
7159 }
7160 if outcomes.iter().all(CommandOutcome::ok) {
7161 break;
7162 }
7163 match self.gate_fix_round(&winner, &outcomes).await? {
7164 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7165 GateFix::Stop => break,
7166 GateFix::Defer => {
7167 self.state.save()?;
7168 return Ok(());
7169 }
7170 }
7171 }
7172 let passed = outcomes.iter().all(CommandOutcome::ok);
7173 self.state.gate = outcomes;
7174 self.state.gate_ran = true;
7175 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7176 if !passed {
7177 self.state.status = RunStatus::Blocked;
7178 let spent = self.state.gate_fixes.len();
7179 self.state.event(
7180 "gate",
7181 if spent == 0 {
7182 "gate failed; not merging".to_owned()
7183 } else {
7184 format!("gate failed after {spent} gate-fix round(s); not merging")
7185 },
7186 );
7187 }
7188 self.state.save()?;
7189 Ok(())
7190 }
7191
7192 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7202 let commands = state.config.verify.pre_gate.clone();
7203 if commands.is_empty() {
7204 return;
7205 }
7206 let shell = state.config.shell();
7207 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7208 let (outcomes, _) = run_commands(
7209 state,
7210 "pre_gate",
7211 "pre_gate",
7212 0,
7213 &shell,
7214 &commands,
7215 &winner.worktree,
7216 timeout,
7217 )
7218 .await;
7219 for o in &outcomes {
7220 if !o.ok() {
7221 tracing::warn!(
7222 "pre_gate `{}` failed ({:?}); the gate decides",
7223 o.command,
7224 o.code
7225 );
7226 }
7227 state.event(
7228 "pre_gate",
7229 format!(
7230 "`{}` -> {}",
7231 o.command,
7232 if o.ok() {
7233 "pass".to_owned()
7234 } else {
7235 format!(
7236 "FAIL ({:?})\n{}",
7237 o.code,
7238 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7239 )
7240 }
7241 ),
7242 );
7243 }
7244 state.pre_gate = outcomes;
7245 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7246 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7247 Ok(head) => {
7248 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7249 state.pre_gate_commit = Some(head);
7250 }
7251 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7252 },
7253 Ok(false) => {}
7254 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7255 }
7256 if let Err(e) = state.save() {
7257 tracing::warn!("could not persist the pre_gate record: {e:#}");
7258 }
7259 }
7260
7261 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7264 let wait = self.repository_wait();
7265 run_landing_gate(&mut self.state, winner, &wait).await
7266 }
7267
7268 async fn gate_fix_round(
7281 &mut self,
7282 winner: &Candidate,
7283 outcomes: &[CommandOutcome],
7284 ) -> Result<GateFix> {
7285 let cap = self.state.config.graph.gate_fix_rounds;
7286 let spent = self.state.gate_fixes.len();
7287 if spent >= cap {
7288 if cap > 0 {
7289 self.state.event(
7290 "gate",
7291 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7292 );
7293 }
7294 return Ok(GateFix::Stop);
7295 }
7296 if !gate_fixable(outcomes) {
7297 self.state.event(
7298 "gate",
7299 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7300 command or similar); not spending a fix round on it",
7301 );
7302 return Ok(GateFix::Stop);
7303 }
7304 let min_free = self.state.config.disk.min_free_bytes;
7305 if min_free > 0 {
7306 match crate::disk::free_bytes(&winner.worktree) {
7307 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7308 Ok(free) => {
7309 self.state.event(
7310 "gate",
7311 format!(
7312 "only {free} bytes free ({min_free} required by `[disk] \
7313 min_free_bytes`); not spending a fix round on a failure the disk \
7314 may explain"
7315 ),
7316 );
7317 return Ok(GateFix::Stop);
7318 }
7319 Err(e) => {
7320 self.state.event(
7321 "gate",
7322 format!("free disk space could not be measured ({e:#}); no fix round"),
7323 );
7324 return Ok(GateFix::Stop);
7325 }
7326 }
7327 }
7328
7329 let attempt = spent + 1;
7330 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7331 let base = self.landing_base();
7332 let fix_prompt = prompt::gate_fix_in(
7333 &self.state.instruction,
7334 &failed,
7335 attempt,
7336 cap,
7337 &self.state.config.graph.language,
7338 &self.state.config.graph.github_language,
7339 );
7340 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7341 let sessions = self.state.config.graph.sessions;
7342 let artifacts = self.merge_repair_artifacts();
7343 self.state.event(
7344 "gate",
7345 format!("gate failed; gate-fix round {attempt} of {cap}"),
7346 );
7347 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7348 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7349 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7352 head: before.clone(),
7353 passed: false,
7354 outcomes: Vec::new(),
7355 });
7356 self.state.save()?;
7357 let (_, seat, out) = self
7358 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7359 prompt: fix_prompt.clone(),
7360 spec,
7361 seat,
7362 cwd: winner.worktree.clone(),
7363 timeout,
7364 allow_write: true,
7365 sessions,
7366 artifacts: artifacts.clone(),
7367 stem: format!("gate-fix-{attempt}"),
7368 handover: None,
7369 })
7370 .await;
7371 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7374 return Ok(GateFix::Defer);
7375 }
7376 let mut record = GateFixRecord {
7377 agent: seat.agent.clone(),
7378 failed,
7379 notes: String::new(),
7380 committed: false,
7381 error: None,
7382 };
7383 match out {
7384 AgentOutcome::Ok(o) => {
7385 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7388 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7389 }
7390 }
7391 AgentOutcome::Dropped(_) => {
7392 record.error = Some("the CLI dropped the stream".to_owned());
7393 }
7394 AgentOutcome::Quota(o) => {
7395 self.state.quota.push(QuotaLoss {
7396 seat: seat.key.clone(),
7397 node: "gate-fix".to_owned(),
7398 at: Timestamp::now(),
7399 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7400 });
7401 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7402 }
7403 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7404 }
7405 self.state.seats.insert(seat.key.clone(), seat);
7406 let coordination = self.coordinate(&winner.worktree).await?;
7407 if let Ok(r) = git::rescue_commit(
7408 &winner.worktree,
7409 &format!("magi: gate fix {attempt} (uncommitted work)"),
7410 )
7411 .await
7412 {
7413 self.state.note_withheld("gate-fix", &r.withheld);
7414 }
7415 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7416 record.committed = after != before;
7417 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7418 let note = record.error.clone();
7419 self.state.gate_fixes.push(record);
7420 self.state.save()?;
7421 if !changed {
7422 self.state.event(
7423 "gate",
7424 match note {
7425 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7426 None => format!("gate-fix round {attempt}: the tree did not change"),
7427 },
7428 );
7429 return Ok(GateFix::Stop);
7430 }
7431 self.state.event(
7432 "gate",
7433 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7434 );
7435
7436 drop(coordination);
7437 let wait = self.repository_wait();
7438 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7439 }
7440
7441 async fn guarded_pr_message(
7448 &mut self,
7449 winner: &Candidate,
7450 facts: Option<&BranchFacts>,
7451 posting: bool,
7452 ) -> Result<Option<PrMessage>> {
7453 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7454 let head = match final_head.as_deref() {
7455 Some(h) => crate::pr_outcome::FinalHead::At(h),
7456 None => crate::pr_outcome::FinalHead::Unknown,
7457 };
7458 let summarized = if posting {
7460 Some(self.summarize_change(winner, final_head.as_deref()).await)
7461 } else {
7462 None
7463 };
7464 if self.state.resource_block.is_some() {
7469 self.state.event(
7470 "pr-summary",
7471 "a shared resource was busy; nothing was published, resume to summarize again",
7472 );
7473 return Ok(None);
7474 }
7475 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7476 anyhow::bail!(
7477 "the final commit moved during public description generation; retry before publishing"
7478 );
7479 }
7480 let note = match (&summarized, &self.state.change_summary) {
7481 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7482 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7483 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7484 (None, _) => ChangeNote::NotAsked,
7485 };
7486 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7487 if !posting {
7488 let identity = crate::scrub::Identity::current();
7489 return Ok(Some(PrMessage {
7490 title: crate::scrub::scrub(&pr.title, &identity),
7491 body: crate::scrub::scrub(&pr.body, &identity),
7492 }));
7493 }
7494 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7497 if let Some(known) = self
7498 .state
7499 .github_text
7500 .clone()
7501 .filter(|g| g.fingerprint == raw_fp)
7502 {
7503 return self.settle_withheld(winner, &pr, known).await;
7504 }
7505 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7506 let mut violations =
7507 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7508 let identity = crate::scrub::Identity::current();
7509 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7510 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7511 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7512 {
7513 violations.push(crate::github_text::Violation::SensitiveData);
7514 }
7515 let has_language = violations
7519 .iter()
7520 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7521 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7522 self.state.event(
7523 "github-text",
7524 if has_language {
7525 format!("description rejected: {violations:?}; requesting one rewrite")
7526 } else {
7527 format!("description rejected: {violations:?}; asking the owner")
7528 },
7529 );
7530 let mut rewritten = false;
7531 let mut rejected_rewrite = None;
7532 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7533 {
7534 let output = text;
7535 if let Some(title) = summary_title(&output) {
7536 let raw_body = summary_without_title(&output);
7539 let clean_title = crate::scrub::scrub(&title, &identity);
7540 let mut body = crate::scrub::scrub(&raw_body, &identity);
7541 let redacted = clean_title != title || body != raw_body;
7542 let title = clean_title;
7543 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7544 if !complete {
7545 self.state.event(
7546 "github-text",
7547 "description rewrite omitted required findings",
7548 );
7549 }
7550 if !body.trim().is_empty() {
7551 body.push_str("\n\n");
7552 body.push_str(&public_pr_evidence(&self.state, head));
7553 }
7554 let marker = format!("magi:run/{}", self.state.id);
7555 if !body.trim().is_empty() && !body.contains(&marker) {
7556 body.push_str(&format!("\n\n{marker}"));
7557 }
7558 let rewrite = PrMessage {
7559 title: title.clone(),
7560 body: body.clone(),
7561 };
7562 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7563 let mut failed = crate::github_text::check_for(
7564 &self.state.config.graph,
7565 &title,
7566 &body,
7567 redecision,
7568 );
7569 if (!crate::github_text::shareable(&title)
7570 || !crate::github_text::shareable(&body))
7571 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7572 {
7573 failed.push(crate::github_text::Violation::SensitiveData);
7574 }
7575 if complete
7576 && !title.trim().is_empty()
7577 && !body.trim().is_empty()
7578 && !failed.is_empty()
7579 {
7580 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7581 title: title.clone(),
7582 body: body.clone(),
7583 failed: failed
7584 .iter()
7585 .map(|v| crate::github_text::category(*v).to_owned())
7586 .collect(),
7587 });
7588 }
7589 if complete
7590 && !title.trim().is_empty()
7591 && !raw_body.trim().is_empty()
7592 && failed.is_empty()
7593 {
7594 decision = redecision;
7595 pr = PrMessage { title, body };
7596 rewritten = true;
7597 self.state
7598 .event("github-text", "description rewrite passed");
7599 if redacted {
7600 self.state.event(
7601 "github-text",
7602 "sensitive data removed from description rewrite",
7603 );
7604 }
7605 }
7606 }
7607 }
7608 if !rewritten {
7609 self.state.event(
7610 "github-text",
7611 "description rewrite unavailable or rejected; asking the owner",
7612 );
7613 if let Some(w) =
7614 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7615 {
7616 self.retire_github_text_question("the withheld text changed");
7619 let record = crate::run::GithubTextAsk {
7620 rewrite: rejected_rewrite,
7621 fingerprint: raw_fp,
7622 title: w.title,
7623 body: w.body,
7624 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7625 question: None,
7626 asks: 0,
7627 resolved: false,
7628 chosen_title: None,
7629 choice: None,
7630 };
7631 return self.settle_withheld(winner, &pr, record).await;
7632 }
7633 }
7634 }
7635 let (title, body) =
7636 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7637 Ok(Some(PrMessage { title, body }))
7638 }
7639
7640 async fn summarize_change(
7651 &mut self,
7652 winner: &Candidate,
7653 head: Option<&str>,
7654 ) -> std::result::Result<(), String> {
7655 let Some(head) = head else {
7656 return Err("the final commit could not be read".to_owned());
7657 };
7658 let remote = self.state.config.merge.remote.clone();
7659 let tracking = format!("{remote}/{}", self.state.base_branch);
7660 if !git::rev_exists(&winner.worktree, &tracking).await {
7661 return Err(format!("the remote base `{tracking}` could not be read"));
7662 }
7663 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7664 Ok(b) if !b.is_empty() => b,
7665 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7666 };
7667 let evidence = crate::pr_outcome::summary_evidence(
7668 &self.state,
7669 crate::pr_outcome::FinalHead::At(head),
7670 );
7671 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7672 if self
7673 .state
7674 .change_summary
7675 .as_ref()
7676 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7677 {
7678 return Ok(());
7679 }
7680 if self.state.change_summary.take().is_some() {
7681 self.state.event(
7682 "pr-summary",
7683 "the final diff or evidence changed; the earlier change summary was discarded",
7684 );
7685 }
7686 if let Err(e) = write_artifact(
7688 &self.state,
7689 &format!("public-description-{fingerprint}.md"),
7690 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7691 ) {
7692 tracing::warn!("could not save public description evidence: {e:#}");
7693 }
7694 let chain = match agent::pick_chain(
7695 &self.state.config.agents,
7696 self.state.config.roles.synthesizer.as_ref(),
7697 &agent::installed,
7698 "synthesizer",
7699 ) {
7700 Ok(chain) => chain,
7701 Err(e) => {
7702 tracing::warn!("no agent for the change summary: {e:#}");
7703 self.state.event("pr-summary", "no summary agent available");
7704 return Err("no agent was available to summarize it".to_owned());
7705 }
7706 };
7707 let subjects = git::subjects(&winner.worktree, &base, head)
7708 .await
7709 .unwrap_or_default();
7710 let stat = git::diff_stat(&winner.worktree, &base, head)
7711 .await
7712 .unwrap_or_default();
7713 let stat: String = stat.chars().take(6_000).collect();
7714 let mut prompt = crate::prompt::change_summary_in(
7715 &base,
7716 head,
7717 &subjects,
7718 stat.trim(),
7719 &self.state.config.graph.github_language,
7720 );
7721 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7722 let sessions = self.state.config.graph.sessions;
7723 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7724 let prompts = self.state.config.prompts.clone();
7725 let cache = self.state.config.cache_dir();
7726 let run = self.state.id.clone();
7727 for (i, spec) in chain.into_iter().enumerate() {
7728 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7729 let github_spec = match agent::for_github_text(
7730 &spec,
7731 &self.state.config.graph.github_language,
7732 &winner.worktree,
7733 ) {
7734 Ok(spec) => spec,
7735 Err(e) => {
7736 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7737 continue;
7738 }
7739 };
7740 let job = SeatJob {
7741 spec: github_spec,
7742 seat,
7743 cwd: winner.worktree.clone(),
7744 prompt: prompt.clone(),
7745 timeout,
7746 allow_write: false,
7747 sessions,
7748 artifacts: agent::artifacts_dir(&self.state.dir()),
7749 stem: if i == 0 {
7750 "change-summary".to_owned()
7751 } else {
7752 format!("change-summary-{}", spec.id)
7753 },
7754 handover: None,
7755 };
7756 let ctx = WaveCtx {
7757 run: &run,
7758 node: "pr-summary",
7759 prompts: &prompts,
7760 cache: cache.as_deref(),
7761 round: None,
7762 carry_seats: false,
7763 pauses: Some((&self.pause, &self.interrupt)),
7764 };
7765 let (seat, outcome) =
7766 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7767 self.state.seats.insert(seat.key.clone(), seat);
7768 let text = match outcome {
7769 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7770 AgentOutcome::ResourceBlocked(why) => {
7771 if self.state.resource_block.is_none() {
7774 self.state.resource_block = Some(crate::run::ResourceBlock {
7775 cause: crate::run::ResourceBlockCause::Other,
7776 node: "pr-summary".to_owned(),
7777 kind: "shared resource".to_owned(),
7778 detail: why,
7779 at: Timestamp::now(),
7780 });
7781 }
7782 return Err("a shared resource was busy".to_owned());
7783 }
7784 _ => continue,
7785 };
7786 let blind = &self.state.config.blind;
7787 let Some(title) = summary_title(&text) else {
7788 continue;
7789 };
7790 let body: String = summary_without_title(&text)
7791 .lines()
7792 .filter(|l| !l.contains("magi:run/"))
7793 .collect::<Vec<_>>()
7794 .join("\n");
7795 let body = blind::sanitize_prose(body.trim(), blind);
7796 let title = blind::sanitize_prose(&title, blind);
7797 if body.trim().is_empty() || title.trim().is_empty() {
7798 continue;
7799 }
7800 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7801 continue;
7802 }
7803 if git::rev_parse(&winner.worktree, "HEAD")
7805 .await
7806 .ok()
7807 .as_deref()
7808 != Some(head)
7809 {
7810 return Err("the branch moved while the summary was written".to_owned());
7811 }
7812 if git::merge_base(&winner.worktree, &tracking, head)
7813 .await
7814 .ok()
7815 .as_deref()
7816 != Some(base.as_str())
7817 || crate::pr_outcome::summary_evidence(
7818 &self.state,
7819 crate::pr_outcome::FinalHead::At(head),
7820 ) != evidence
7821 {
7822 return Err("the summary evidence changed while it was written".to_owned());
7823 }
7824 self.state.change_summary = Some(crate::run::ChangeSummary {
7825 head: head.to_owned(),
7826 base,
7827 evidence: fingerprint,
7828 title,
7829 body,
7830 });
7831 self.state.event(
7832 "pr-summary",
7833 format!("change summary written by {} for the final diff", spec.id),
7834 );
7835 if let Err(e) = self.state.save() {
7836 tracing::warn!("could not save the change summary: {e:#}");
7837 }
7838 return Ok(());
7839 }
7840 self.state
7841 .event("pr-summary", "no agent produced a change summary");
7842 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7843 }
7844
7845 async fn rewrite_pr_text(
7855 &mut self,
7856 winner: &Candidate,
7857 pr: &PrMessage,
7858 violations: &[crate::github_text::Violation],
7859 ) -> Option<String> {
7860 let prompt = format!(
7861 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7862 pr.title, pr.body
7863 );
7864 let sessions = self.state.config.graph.sessions;
7865 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7866 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7869 let key = format!("impl-{}", winner.label);
7870 if let Some(spec) = self
7871 .state
7872 .config
7873 .agents
7874 .iter()
7875 .find(|a| a.id == winner.agent)
7876 .cloned()
7877 && let Some(existing) = self.state.seats.get(&key)
7878 && existing.agent == spec.id
7879 && agent::has_session(spec.kind, existing, sessions)
7880 {
7881 attempts.push((spec, Some(existing.clone())));
7882 }
7883 match agent::pick_chain(
7884 &self.state.config.agents,
7885 self.state.config.roles.language_judge.as_ref(),
7886 &agent::installed,
7887 "language_judge",
7888 ) {
7889 Ok(chain) => {
7890 for spec in chain {
7891 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7892 continue;
7893 }
7894 attempts.push((spec, None));
7895 }
7896 }
7897 Err(e) if attempts.is_empty() => {
7898 tracing::warn!("no agent for the description rewrite: {e:#}");
7899 self.state
7900 .event("github-text", "no rewrite agent available");
7901 return None;
7902 }
7903 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7904 }
7905 let prompts = self.state.config.prompts.clone();
7906 let cache = self.state.config.cache_dir();
7907 let run = self.state.id.clone();
7908 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7909 let resumed = resumed_seat.is_some();
7910 let seat = resumed_seat.unwrap_or_else(|| {
7911 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7912 });
7913 let github_spec = match agent::for_github_text(
7914 &spec,
7915 &self.state.config.graph.github_language,
7916 &winner.worktree,
7917 ) {
7918 Ok(spec) => spec,
7919 Err(e) => {
7920 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7921 continue;
7922 }
7923 };
7924 let job = SeatJob {
7925 spec: github_spec,
7926 seat,
7927 cwd: winner.worktree.clone(),
7928 prompt: prompt.clone(),
7929 timeout: retry_budget(timeout_full, resumed),
7930 allow_write: false,
7931 sessions,
7932 artifacts: agent::artifacts_dir(&self.state.dir()),
7933 stem: if i == 0 {
7934 "github-text-rewrite".to_owned()
7935 } else {
7936 format!("github-text-rewrite-{}", spec.id)
7937 },
7938 handover: None,
7939 };
7940 let ctx = WaveCtx {
7941 run: &run,
7942 node: "github-text",
7943 prompts: &prompts,
7944 cache: cache.as_deref(),
7945 round: None,
7946 carry_seats: false,
7947 pauses: Some((&self.pause, &self.interrupt)),
7948 };
7949 let (seat, outcome) =
7950 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7951 self.state.seats.insert(seat.key.clone(), seat);
7952 match outcome {
7953 AgentOutcome::Ok(o)
7954 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7955 {
7956 return Some(o.text);
7957 }
7958 AgentOutcome::ResourceBlocked(_) => return None,
7959 _ => {}
7960 }
7961 }
7962 None
7963 }
7964
7965 fn retire_github_text_question(&mut self, why: &str) {
7967 let Some(id) = self
7968 .state
7969 .github_text
7970 .as_ref()
7971 .and_then(|g| g.question.clone())
7972 else {
7973 return;
7974 };
7975 let _ = ask::Questions::open().update(&id, |q| {
7976 q.abandon(why);
7977 Ok(())
7978 });
7979 }
7980
7981 async fn settle_withheld(
7985 &mut self,
7986 winner: &Candidate,
7987 pr: &PrMessage,
7988 mut g: crate::run::GithubTextAsk,
7989 ) -> Result<Option<PrMessage>> {
7990 use crate::github_text as gt;
7991 let store = ask::Questions::open();
7992 let timeout = self.state.config.graph.answer_timeout;
7993 if !g.resolved {
7994 let mut standing = g
7995 .question
7996 .as_ref()
7997 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7998 if let Some(q) = standing.as_ref()
7999 && q.status.open()
8000 && gt::expired(q, timeout)
8001 {
8002 let why = format!("no answer within {}s of asking", timeout.max(1));
8003 standing = store
8004 .update(&q.id, |q| {
8005 q.abandon(&why);
8006 Ok(())
8007 })
8008 .ok()
8009 .map(|(q, ())| q);
8010 }
8011 let mut ask_again: Option<Option<Vec<String>>> = None;
8012 match standing {
8013 None if g.asks <= 1 => {
8016 g.asks = 0;
8017 ask_again = Some(None);
8018 }
8019 None => {
8020 g.resolved = true;
8021 g.chosen_title = None;
8022 g.choice = Some(GithubTextChoice::Fallback);
8023 }
8024 Some(q) if q.status.open() => {
8025 self.park_on_github_text(&g)?;
8026 return Ok(None);
8027 }
8028 Some(q) => match gt::read_reply(&q) {
8029 gt::Reply::Fallback => {
8030 g.resolved = true;
8031 g.chosen_title = None;
8032 g.choice = Some(GithubTextChoice::Fallback);
8033 }
8034 gt::Reply::Original => {
8035 g.resolved = true;
8036 g.chosen_title = None;
8037 g.choice = Some(GithubTextChoice::Original);
8038 }
8039 gt::Reply::Rewrite => {
8040 g.resolved = true;
8041 g.chosen_title = None;
8042 g.choice = Some(if g.rewrite.is_some() {
8043 GithubTextChoice::Rewrite
8044 } else {
8045 GithubTextChoice::Fallback
8046 });
8047 }
8048 gt::Reply::Title(text) => {
8049 let first = text
8050 .lines()
8051 .map(str::trim)
8052 .find(|l| !l.is_empty())
8053 .unwrap_or("");
8054 let decision = if first.is_empty() {
8055 None
8056 } else {
8057 let candidate = PrMessage {
8058 title: first.to_owned(),
8059 body: String::new(),
8060 };
8061 self.judge_pr_language(&winner.worktree, &candidate).await
8062 };
8063 match gt::vet_title(
8064 &text,
8065 decision,
8066 self.state.config.graph.github_english(),
8067 ) {
8068 Ok(title) => {
8069 g.resolved = true;
8070 g.chosen_title = Some(title);
8071 g.choice = Some(GithubTextChoice::MyTitle);
8072 }
8073 Err(cats) if g.asks < 2 => {
8074 ask_again = Some(Some(
8075 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8076 ))
8077 }
8078 Err(_) => {
8079 g.resolved = true;
8080 g.chosen_title = None;
8081 g.choice = Some(GithubTextChoice::Fallback);
8082 }
8083 }
8084 }
8085 },
8086 }
8087 if let Some(rejected) = ask_again {
8088 let w = gt::Withheld {
8091 title: g.title,
8092 body: g.body,
8093 categories: g.categories.clone(),
8094 };
8095 let artifact = match crate::run::write_artifact(
8098 &self.state,
8099 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8100 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8101 ) {
8102 Ok(p) => Some(p),
8103 Err(e) => {
8104 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8105 None
8106 }
8107 };
8108 let (detail, comparison) = gt::question_content(
8109 &w,
8110 &pr.title,
8111 &pr.body,
8112 &self.state.id,
8113 rejected.as_deref(),
8114 artifact.as_deref(),
8115 g.rewrite.as_ref(),
8116 );
8117 let mut q = ask::Question::new(
8118 self.state.id.clone(),
8119 gt::ASK_NODE.to_owned(),
8120 gt::ASK_SEAT.to_owned(),
8121 gt::question_summary(&self.state.config.graph.language, &w),
8122 detail,
8123 gt::question_choices(&w, g.rewrite.as_ref()),
8124 );
8125 q.posting_comparison = Some(comparison);
8126 q.answer_timeout = timeout;
8127 g.question = Some(q.id.clone());
8130 g.asks += 1;
8131 self.state.github_text = Some(g.clone());
8132 self.state.event(
8133 "github-text",
8134 format!("asking the owner about the withheld text ({})", q.short()),
8135 );
8136 self.state.save()?;
8137 store
8138 .put_executable(&mut q)
8139 .context("file the posting-gate question")?;
8140 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8141 tracing::warn!(
8142 "could not notify about posting-gate question {}: {e:#}",
8143 q.short()
8144 );
8145 }
8146 self.park_on_github_text(&g)?;
8147 return Ok(None);
8148 }
8149 }
8150 let resolved = g.chosen_title.clone();
8151 let original = g.choice == Some(GithubTextChoice::Original);
8152 self.state.event(
8154 "github-text",
8155 if original {
8156 format!(
8157 "posting the original text for: {}",
8158 match (g.title, g.body) {
8159 (true, true) => "title, description",
8160 (true, false) => "title",
8161 _ => "description",
8162 }
8163 )
8164 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8165 "posting the saved rewrite title and description".to_owned()
8166 } else if resolved.is_some() {
8167 "posting the owner's replacement title".to_owned()
8168 } else {
8169 "posting the neutral text for the withheld fields".to_owned()
8170 },
8171 );
8172 self.state.github_text = Some(g.clone());
8173 self.state.save()?;
8174 if g.choice == Some(GithubTextChoice::Rewrite)
8175 && let Some(r) = g.rewrite.as_ref()
8176 {
8177 return Ok(Some(PrMessage {
8178 title: r.title.clone(),
8179 body: r.body.clone(),
8180 }));
8181 }
8182 let w = gt::Withheld {
8183 title: g.title,
8184 body: g.body,
8185 categories: g.categories.clone(),
8186 };
8187 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8188 if let Some(t) = resolved {
8189 title = t;
8190 }
8191 let english = gt::LanguageDecision {
8193 title_english: true,
8194 body_english: true,
8195 body_complete: true,
8196 };
8197 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8198 if original {
8199 if g.title {
8202 title = pr.title.clone();
8203 }
8204 if g.body {
8205 body = pr.body.clone();
8206 }
8207 }
8208 Ok(Some(PrMessage { title, body }))
8209 }
8210
8211 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8212 self.state.github_text = Some(g.clone());
8213 self.state.end_pass_parked(ParkCause::Question);
8214 self.state.parked = true;
8215 self.state.event(
8216 "github-text",
8217 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8218 );
8219 self.state.save()
8220 }
8221
8222 async fn judge_pr_language(
8225 &mut self,
8226 cwd: &Path,
8227 pr: &PrMessage,
8228 ) -> Option<crate::github_text::LanguageDecision> {
8229 self.state.config.roles.language_judge.as_ref()?;
8230 if !self.state.config.graph.language_gate() {
8231 return None;
8232 }
8233 let decision =
8234 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8235 self.state.event(
8236 "github-text",
8237 if decision.is_some() {
8238 "language decided by the language judge"
8239 } else {
8240 "language judge unavailable; using built-in heuristics"
8241 },
8242 );
8243 decision
8244 }
8245
8246 async fn merge(&mut self) -> Result<()> {
8247 if self
8262 .state
8263 .base_sync
8264 .as_ref()
8265 .is_some_and(|s| s.conflict.is_some())
8266 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8267 != Some(RunStatus::Gating)
8268 || !self.state.gate_allows_progress()
8276 {
8277 return Ok(());
8278 }
8279 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8288 return Ok(());
8289 }
8290 let Some(winner) = self.state.winner().cloned() else {
8291 return Ok(());
8292 };
8293 let repo = self.state.repo.clone();
8294 let base = self.state.base_branch.clone();
8295 let mode = self.state.config.merge.mode;
8296 let style = self.state.config.merge.style;
8297 let facts = if is_review_run(&self.state) {
8298 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8299 let start = review_base(
8300 &repo,
8301 &self.state.config.merge.remote,
8302 &base,
8303 &self.state.base_commit,
8304 &winner.branch,
8305 )
8306 .await;
8307 branch_facts(&repo, &start, &winner.branch).await
8308 } else {
8309 None
8310 };
8311 let leaked = if is_review_run(&self.state) {
8314 leaked_subjects(&self.state, &winner.branch).await
8315 } else {
8316 Some(Vec::new())
8317 };
8318 let Some(pr) = self
8322 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8323 .await?
8324 else {
8325 return Ok(());
8326 };
8327 let message = pr.commit_message();
8328
8329 let outcome = match mode {
8330 MergeMode::None => MergeOutcome {
8331 mode,
8332 ok: true,
8333 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8334 empty: false,
8335 },
8336 MergeMode::Pr | MergeMode::Local
8337 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8338 {
8339 MergeOutcome {
8340 mode,
8341 ok: false,
8342 detail: empty_candidate_detail(&self.state, &base),
8343 empty: true,
8344 }
8345 }
8346 MergeMode::Local => {
8347 let on = git::current_branch(&repo).await?;
8348 if on.as_deref() != Some(base.as_str()) {
8349 MergeOutcome {
8350 mode,
8351 ok: false,
8352 detail: format!(
8353 "{} has {} checked out, not the base branch {base}",
8354 repo.display(),
8355 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8356 ),
8357 empty: false,
8358 }
8359 } else if !git::is_clean(&repo).await? {
8360 MergeOutcome {
8361 mode,
8362 ok: false,
8363 detail: format!("{} is dirty; refusing to merge", repo.display()),
8364 empty: false,
8365 }
8366 } else {
8367 let out = match style {
8368 MergeStyle::Merge => {
8369 git::merge_no_ff(&repo, &winner.branch, &message).await?
8370 }
8371 MergeStyle::Squash => {
8372 git::merge_squash(&repo, &winner.branch, &message).await?
8373 }
8374 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8375 };
8376 MergeOutcome {
8377 mode,
8378 ok: out.ok(),
8379 detail: if out.ok() { out.stdout } else { out.stderr },
8380 empty: false,
8381 }
8382 }
8383 }
8384 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8385 self.state.pr_publication_pending = true;
8386 self.state.save()?;
8387 let out =
8388 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8389 self.state.pr_publication_pending = out.is_err();
8390 MergeOutcome {
8391 mode,
8392 ok: out.is_ok(),
8393 detail: match out {
8394 Ok(url) => url,
8395 Err(e) => format!("{e:#}"),
8396 },
8397 empty: false,
8398 }
8399 }
8400 MergeMode::Pr => {
8401 let remote = self.state.config.merge.remote.clone();
8402 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8403 if !pushed.ok() {
8404 MergeOutcome {
8405 mode,
8406 ok: false,
8407 detail: pushed.stderr,
8408 empty: false,
8409 }
8410 } else {
8411 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8417 let out = match pr_merge_plan(found) {
8418 PrPlan::Create => {
8419 gh_pr_create(
8420 &winner.worktree,
8421 &base,
8422 &winner.branch,
8423 &pr.title,
8424 &pr.body,
8425 )
8426 .await
8427 }
8428 PrPlan::Adopt { url, title } => {
8429 self.state
8430 .event("merge", format!("Pr: adopted open pull request {url}"));
8431 if self.state.github_text.as_ref().is_some_and(|g| {
8432 g.resolved
8433 && g.choice == Some(GithubTextChoice::Rewrite)
8434 && g.rewrite.is_some()
8435 }) {
8436 land::set_pr_text_verbatim(
8439 &winner.worktree,
8440 &url,
8441 &pr.title,
8442 &pr.body,
8443 )
8444 .await
8445 .map(|()| url)
8446 } else {
8447 if title != pr.title
8448 && (!is_review_run(&self.state)
8449 || leaked
8450 .as_deref()
8451 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8452 && let Err(e) =
8453 if self.state.github_text.as_ref().is_some_and(|g| {
8454 g.resolved
8455 && g.title
8456 && g.choice == Some(GithubTextChoice::Original)
8457 }) {
8458 land::set_pr_title_verbatim(
8460 &winner.worktree,
8461 &url,
8462 &pr.title,
8463 )
8464 .await
8465 } else {
8466 land::set_pr_title(
8467 &mut self.state,
8468 &winner.worktree,
8469 &url,
8470 &pr.title,
8471 )
8472 .await
8473 }
8474 {
8475 tracing::warn!("could not refresh title of {url}: {e:#}");
8476 self.state
8477 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8478 }
8479 Ok(url)
8480 }
8481 }
8482 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8483 };
8484 match out {
8485 Ok(url) => MergeOutcome {
8486 mode,
8487 ok: true,
8488 detail: url,
8489 empty: false,
8490 },
8491 Err(e) => MergeOutcome {
8492 mode,
8493 ok: false,
8494 detail: e.to_string(),
8495 empty: false,
8496 },
8497 }
8498 }
8499 }
8500 };
8501
8502 self.state.status = match (mode, outcome.ok) {
8503 (MergeMode::None, _) => RunStatus::Ready,
8504 (_, true) if outcome.empty => RunStatus::Ready,
8505 (MergeMode::Pr, true) => RunStatus::Ready,
8506 (_, true) => RunStatus::Merged,
8507 (_, false) => RunStatus::Blocked,
8508 };
8509 self.state.event(
8510 "merge",
8511 format!(
8512 "{:?}: {}",
8513 mode,
8514 outcome.detail.lines().next().unwrap_or("")
8515 ),
8516 );
8517 self.state.merge = Some(outcome);
8518 self.state.save()?;
8519
8520 if self.state.config.graph.land
8526 && mode == MergeMode::Pr
8527 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8528 {
8529 self.run_land().await?;
8530 }
8531 self.settle_questions();
8536 Ok(())
8537 }
8538
8539 async fn run_land(&mut self) -> Result<()> {
8550 crate::pr_head::ensure_owner(&self.state)?;
8551 let url = self
8552 .state
8553 .pr
8554 .as_ref()
8555 .map(|pr| pr.url.clone())
8556 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8557 .unwrap_or_default();
8558 let url = url.lines().next().unwrap_or("").trim().to_owned();
8559 if !url.starts_with("http") {
8560 return Ok(());
8561 }
8562 loop {
8565 match land::land(&mut self.state, &url).await {
8566 Ok(_)
8567 if self
8568 .state
8569 .merge_repairs
8570 .last()
8571 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8572 {
8573 if self.repair_merge_review().await? {
8574 continue;
8575 }
8576 }
8577 Ok(pr) if self.state.parked => {
8578 let _ = pr;
8582 }
8583 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8584 self.state.status = RunStatus::Superseded;
8585 self.state.save()?;
8586 }
8587 Ok(pr) => {
8588 self.state.status = match pr.state {
8589 land::PrLifecycle::Merged => RunStatus::Merged,
8590 _ => RunStatus::Blocked,
8591 };
8592 if bump::should_release_bump(self.state.status)
8599 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8600 {
8601 self.state
8607 .event("bump", format!("release bump skipped: {e:#}"));
8608 }
8609 if self.state.status == RunStatus::Merged {
8612 crate::followup::after_merge(&mut self.state, &pr.url).await;
8613 }
8614 self.state.save()?;
8615 }
8616 Err(e) => {
8617 if e.is::<crate::repo_lock::ResourceBlocked>() {
8618 return Err(e);
8619 }
8620 self.state.status = RunStatus::Blocked;
8621 self.state.event("land", format!("gave up: {e}"));
8622 self.state.save()?;
8623 }
8624 }
8625 break;
8626 }
8627 Ok(())
8628 }
8629
8630 async fn repair_merge_review(&mut self) -> Result<bool> {
8633 use crate::run::MergeRepairPhase as Phase;
8634 crate::pr_head::ensure_owner(&self.state)?;
8635 let i = self.state.merge_repairs.len() - 1;
8636 let request = self.state.merge_repairs[i].clone();
8637 if request.phase == Phase::Fixing {
8638 let restored = match self.state.winner().cloned() {
8639 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8640 None => Err(anyhow::anyhow!("repair has no winning branch")),
8641 };
8642 let why = match restored {
8643 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8644 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8645 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8646 };
8647 self.hold_merge_repair(i, why)?;
8648 return Ok(false);
8649 }
8650 let failure = if i >= self.state.config.graph.land_rounds
8651 || self.state.config.graph.review_rounds == 0
8652 {
8653 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8654 } else {
8655 None
8656 };
8657 if let Some(why) = failure {
8658 self.hold_merge_repair(i, why)?;
8659 return Ok(false);
8660 }
8661 if request.phase == Phase::Requested {
8662 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8663 Ok(pr) => pr,
8664 Err(e) => {
8665 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8666 return Ok(false);
8667 }
8668 };
8669 self.state.merge_repairs[i].phase = Phase::Fixing;
8670 self.state.land_approval = None;
8671 self.state.land_armed_head = None;
8672 self.state.save()?;
8673 let evidence = serde_json::to_string_pretty(&request)?;
8676 let reason = format!(
8677 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8678 );
8679 let budget = self.state.config.graph.land_rounds;
8680 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8681 match result {
8682 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8683 Ok(land::Fixed::Failed(why)) => {
8684 self.hold_merge_repair(
8685 i,
8686 format!(
8687 "repair failed: {why}; inspect the branch and run an explicit review"
8688 ),
8689 )?;
8690 return Ok(false);
8691 }
8692 Err(e) => {
8693 self.hold_merge_repair(
8694 i,
8695 format!(
8696 "repair failed: {e:#}; inspect the branch and run an explicit review"
8697 ),
8698 )?;
8699 return Ok(false);
8700 }
8701 }
8702 self.begin_merge_repair_review(i)?;
8706 }
8707 self.review_loop().await?;
8708 self.gate().await?;
8709 if self.resource_halt()? || self.state.parked {
8710 return Ok(false);
8711 }
8712 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8713 != Some(RunStatus::Gating)
8714 || !self.state.gate_allows_progress()
8715 {
8716 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8717 return Ok(false);
8718 }
8719 let winner = self
8722 .state
8723 .winner()
8724 .cloned()
8725 .context("repair has no winning branch")?;
8726 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8727 if self.state.gate_head.as_deref() != Some(after.as_str())
8728 || !git::is_clean(&winner.worktree).await?
8729 {
8730 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8731 return Ok(false);
8732 }
8733 let expected = self
8734 .state
8735 .pr_pushed_head
8736 .as_deref()
8737 .unwrap_or(&request.head)
8738 .to_owned();
8739 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8740 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8743 let refspec = format!("{after}:refs/heads/{public}");
8744 let push = git::git_raw(
8745 &winner.worktree,
8746 &["push", &lease, &self.state.config.merge.remote, &refspec],
8747 )
8748 .await;
8749 match push {
8750 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8751 Ok(out) => {
8752 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8753 return Ok(false);
8754 }
8755 Err(e) => {
8756 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8757 return Ok(false);
8758 }
8759 }
8760 self.state.merge_repairs[i].phase = Phase::Complete;
8761 self.state.status = RunStatus::Landing;
8762 self.state.event(
8763 "repair",
8764 "new review and gate completed; fresh owner merge decision required",
8765 );
8766 self.state.save()?;
8767 Ok(true)
8768 }
8769
8770 fn merge_repair_artifacts(&self) -> PathBuf {
8771 let artifacts = agent::artifacts_dir(&self.state.dir());
8772 match self
8773 .state
8774 .merge_repairs
8775 .last()
8776 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8777 {
8778 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8779 None => artifacts,
8780 }
8781 }
8782
8783 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8784 self.state.reviews.clear();
8785 self.state.gate.clear();
8786 self.state.gate_ran = false;
8787 self.state.gate_head = None;
8788 self.state.gate_fixes.clear();
8789 self.state.gate_repair_e2e = None;
8790 self.state.pre_gate.clear();
8791 self.state.pre_gate_commit = None;
8792 self.state.contested_handoff = None;
8793 self.state.land_reruns = None;
8794 self.state.land_approval = None;
8795 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8796 self.state.status = RunStatus::Reviewing;
8797 self.state.save()
8798 }
8799
8800 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8801 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8802 self.state.merge_repairs[index].failure = Some(why.clone());
8803 self.state.status = RunStatus::Blocked;
8804 self.state.event("repair", why);
8805 self.state.save()
8806 }
8807
8808 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8812 if let Some(existing) = self.state.seats.get(key)
8813 && existing.agent == agent
8814 {
8815 return existing.clone();
8816 }
8817 let fresh = if self.state.seats.contains_key(key) {
8820 handover_seat(key, agent, self.state.next_seat_seed())
8821 } else {
8822 SeatState::new(key, agent, self.state.seed)
8823 };
8824 self.state.seats.insert(key.to_owned(), fresh.clone());
8825 fresh
8826 }
8827
8828 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8834 match self.state.seats.get(key) {
8835 Some(s) if s.agent != spec.id => {
8836 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8837 }
8838 _ => spec,
8839 }
8840 }
8841
8842 fn view(&self, c: &Candidate) -> CandidateView {
8844 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8845 .unwrap_or_default();
8846 let (patch, _) = blind::sanitize_patch(
8847 &format!("candidate {} patch", c.label),
8848 &raw,
8849 &self.state.config.blind,
8850 );
8851 CandidateView {
8852 label: c.label,
8853 branch: c.branch.clone(),
8854 summary: c.summary.clone(),
8855 stat: c.stat.clone(),
8856 patch,
8857 }
8858 }
8859
8860 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8862 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8863 prompt::judge(
8864 "(see above)",
8865 &views,
8866 self.roles.judges.len(),
8867 base_short,
8868 "en",
8869 )
8870 }
8871
8872 fn vote_prompt_full(
8878 &self,
8879 j: usize,
8880 labels: &[char],
8881 language: &str,
8882 candidates: &[Candidate],
8883 base_short: &str,
8884 ) -> String {
8885 let mut text = format!(
8886 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8887 prompt::final_vote(labels, language),
8888 self.state.instruction,
8889 self.candidate_block(candidates, base_short)
8890 );
8891 if let Some(own) = self
8892 .state
8893 .judgements
8894 .get(j)
8895 .filter(|r| !r.ranking.is_empty())
8896 {
8897 let reasons = own
8898 .reasons
8899 .iter()
8900 .map(|(k, v)| format!("- {k}: {v}"))
8901 .collect::<Vec<_>>()
8902 .join("\n");
8903 text.push_str(&format!(
8904 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8905 own.ranking.iter().collect::<String>(),
8906 if reasons.is_empty() {
8907 ""
8908 } else {
8909 ", because:\n"
8910 }
8911 ));
8912 }
8913 if !self.state.deliberation.is_empty() {
8914 text.push_str("\n# What was argued before this vote\n");
8915 for t in self.transcript(&[], j) {
8916 text.push_str(&format!(
8917 "\n## {}{}\n\n{}\n",
8918 t.who,
8919 if t.is_self { " (you)" } else { "" },
8920 t.body.trim()
8921 ));
8922 }
8923 }
8924 text
8925 }
8926
8927 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8934 let mut turns = Vec::new();
8935 for j in &self.state.judgements {
8936 if j.ranking.is_empty() {
8937 continue;
8938 }
8939 let reasons = j
8940 .reasons
8941 .iter()
8942 .map(|(k, v)| format!("- {k}: {v}"))
8943 .collect::<Vec<_>>()
8944 .join("\n");
8945 turns.push(Turn {
8946 who: format!("Judge {} (opening ranking)", j.judge),
8947 is_self: j.judge == self_idx + 1,
8948 body: format!(
8949 "Ranked {}{}{reasons}",
8950 j.ranking.iter().collect::<String>(),
8951 if reasons.is_empty() {
8952 ""
8953 } else {
8954 ", because:\n"
8955 }
8956 ),
8957 });
8958 }
8959 for t in self
8960 .state
8961 .deliberation
8962 .iter()
8963 .flat_map(|r| r.turns.iter())
8964 .chain(current)
8965 {
8966 turns.push(Turn {
8967 who: format!("Judge {}", t.judge),
8968 is_self: t.judge == self_idx + 1,
8969 body: t.body.clone(),
8970 });
8971 }
8972 turns
8973 }
8974}
8975
8976fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8978 agent::has_session(spec.kind, seat, sessions)
8979}
8980
8981fn next_untried_in_roster<'a>(
9002 roster: &'a [AgentSpec],
9003 start: usize,
9004 tried: &BTreeSet<String>,
9005) -> Option<&'a AgentSpec> {
9006 roster
9007 .get(start + 1..)?
9008 .iter()
9009 .find(|s| !tried.contains(&s.id))
9010}
9011
9012fn pick_successor<'a>(
9033 roster: &'a [AgentSpec],
9034 start: usize,
9035 tried: &BTreeSet<String>,
9036 carried: Option<&BTreeSet<String>>,
9037 others: &BTreeSet<String>,
9038) -> Option<&'a AgentSpec> {
9039 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9040 roster
9041 .get(start + 1..)
9042 .and_then(|tail| {
9043 tail.iter()
9044 .filter(free)
9045 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9046 })
9047 .or_else(|| {
9048 carried
9049 .is_some()
9050 .then(|| roster.iter().find(free))
9051 .flatten()
9052 })
9053 .or_else(|| match carried {
9054 Some(c) => next_for_seat(roster, start, tried, c),
9055 None => next_untried_in_roster(roster, start, tried),
9056 })
9057}
9058
9059fn next_for_seat<'a>(
9069 roster: &'a [AgentSpec],
9070 start: usize,
9071 round_tried: &BTreeSet<String>,
9072 carried_failed: &BTreeSet<String>,
9073) -> Option<&'a AgentSpec> {
9074 roster
9075 .get(start + 1..)?
9076 .iter()
9077 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9078 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9079}
9080
9081fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9087 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9088 return spec;
9089 };
9090 let ok = |id: &str| !h.failed.contains(id);
9091 if let Some(last) = h.last_ok.as_deref()
9092 && ok(last)
9093 && let Some(s) = roster.iter().find(|s| s.id == last)
9094 {
9095 return s.clone();
9096 }
9097 if ok(&spec.id) {
9098 return spec;
9099 }
9100 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9101 roster
9102 .get(from + 1..)
9103 .into_iter()
9104 .flatten()
9105 .chain(roster.iter())
9106 .find(|s| ok(&s.id))
9107 .cloned()
9108 .unwrap_or(spec)
9109}
9110
9111pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9115 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9116}
9117
9118const TIMED_OUT: &str = "timed out";
9121
9122impl FailClass {
9123 fn of(out: &AgentOutcome) -> Option<Self> {
9125 match out {
9126 AgentOutcome::Ok(_) => None,
9127 AgentOutcome::Quota(_) => Some(Self::Quota),
9128 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9129 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9130 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9131 AgentOutcome::ResourceBlocked(_) => None,
9134 }
9135 }
9136
9137 fn stem_word(&self) -> &'static str {
9139 match self {
9140 Self::Quota => "quota",
9141 _ => "handover",
9142 }
9143 }
9144}
9145
9146fn failure_signature(msg: &str) -> String {
9150 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9151 if let Some(code) = line
9154 .strip_prefix("process exited with no usable output (exit code ")
9155 .and_then(|s| s.strip_suffix(')'))
9156 .and_then(|s| s.parse::<i32>().ok())
9157 {
9158 line = format!("exited with some({code}) and no usable output");
9159 } else if matches!(
9160 line.as_str(),
9161 "process exited with no usable output (no exit code reported)"
9162 | "process exited with no usable output (exit code unavailable)"
9163 ) {
9164 line = "exited with none and no usable output".to_owned();
9165 }
9166 let mut out = Vec::new();
9167 for word in line.split_whitespace() {
9168 if word.contains('/') || word.contains('\\') {
9169 out.push("<path>".to_owned());
9170 continue;
9171 }
9172 let mut w = String::new();
9173 let mut in_digits = false;
9174 for c in word.chars() {
9175 if c.is_ascii_digit() {
9176 if !in_digits {
9177 w.push('#');
9178 }
9179 in_digits = true;
9180 } else {
9181 in_digits = false;
9182 w.push(c);
9183 }
9184 }
9185 out.push(w);
9186 }
9187 out.join(" ").chars().take(120).collect()
9188}
9189
9190fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9197 match cur {
9198 FailClass::Quota | FailClass::Timeout => true,
9199 FailClass::Other(_) => prev != Some(cur),
9200 }
9201}
9202
9203fn fail_reason(out: &AgentOutcome) -> String {
9205 match out {
9206 AgentOutcome::Ok(_) => String::new(),
9207 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9208 AgentOutcome::Dropped(o) => format!(
9209 "the CLI dropped the stream ({})",
9210 o.dropped
9211 .as_ref()
9212 .map(|d| d.why.as_str())
9213 .unwrap_or("it ended without delivering its answer")
9214 ),
9215 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9216 e.lines().next().unwrap_or("").chars().take(160).collect()
9217 }
9218 }
9219}
9220
9221pub(crate) fn record_handover(
9224 state: &mut RunState,
9225 node: &str,
9226 seat: &str,
9227 from: &str,
9228 to: &str,
9229 class: &FailClass,
9230 reason: &str,
9231) {
9232 let message = if *class == FailClass::Quota {
9233 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9234 } else {
9235 format!("{seat}: handed over {from} -> {to} ({reason})")
9236 };
9237 state.event(node, message);
9238 state.handovers.push(Handover {
9239 at: Timestamp::now(),
9240 node: node.to_owned(),
9241 seat: seat.to_owned(),
9242 from: from.to_owned(),
9243 to: to.to_owned(),
9244 reason: reason.to_owned(),
9245 });
9246}
9247
9248fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9262 commands.iter().any(|c| c.exit_code.is_none())
9263}
9264
9265fn verified_noop_claim(
9278 usable: bool,
9279 commands: &[agent::CommandEvidence],
9280 text: &str,
9281) -> Option<String> {
9282 (usable && !has_unconfirmed_command(commands))
9283 .then(|| verdict::verified_noop(text))
9284 .flatten()
9285}
9286
9287fn short(commit: &str) -> String {
9288 commit.chars().take(7).collect()
9289}
9290
9291fn make_executable(path: &Path) -> Result<()> {
9292 #[cfg(unix)]
9293 {
9294 use std::os::unix::fs::PermissionsExt as _;
9295 let mut perms = std::fs::metadata(path)?.permissions();
9296 perms.set_mode(0o755);
9297 std::fs::set_permissions(path, perms)?;
9298 }
9299 #[cfg(not(unix))]
9300 {
9301 let _ = path;
9302 }
9303 Ok(())
9304}
9305
9306struct WaveCtx<'a> {
9313 run: &'a str,
9316 node: &'a str,
9318 prompts: &'a Prompts,
9319 cache: Option<&'a Path>,
9321 round: Option<usize>,
9324 carry_seats: bool,
9328 pauses: Option<(&'a Pause, &'a Pause)>,
9331}
9332
9333fn repository_wait(
9335 state: &RunState,
9336 pauses: Option<(&Pause, &Pause)>,
9337) -> crate::repo_lock::WaitPolicy {
9338 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9339 let inherited = crate::repo_lock::WaitPolicy::current();
9340 crate::repo_lock::WaitPolicy::new(
9341 Duration::from_secs(state.config.graph.timeout_resource_wait),
9342 move || {
9343 inherited.cancelled()
9344 || pauses
9345 .as_ref()
9346 .is_some_and(|(a, b)| a.parked() || b.parked())
9347 },
9348 )
9349}
9350
9351fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9352 if error.is::<crate::repo_lock::ResourceBlocked>() {
9353 state.resource_block = Some(crate::run::ResourceBlock {
9354 cause: crate::run::ResourceBlockCause::Other,
9355 node: "repository".into(),
9356 kind: if error.to_string().contains("disk pressure") {
9357 "disk pressure"
9358 } else {
9359 "repository coordination"
9360 }
9361 .into(),
9362 detail: error.to_string(),
9363 at: Timestamp::now(),
9364 });
9365 state.event("resource", error.to_string());
9366 }
9367}
9368
9369async fn coordinate_operation(
9370 state: &mut RunState,
9371 cwd: &Path,
9372 wait: &crate::repo_lock::WaitPolicy,
9373) -> Result<Option<crate::repo_lock::SeatLease>> {
9374 let result =
9375 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9376 .await;
9377 if let Err(e) = &result {
9378 record_repository_block(state, e);
9379 }
9380 result
9381}
9382
9383async fn run_one(
9385 job: SeatJob,
9386 sem: Arc<Semaphore>,
9387 ctx: &WaveCtx<'_>,
9388 state: &mut RunState,
9389 attempt: usize,
9390) -> (SeatState, AgentOutcome) {
9391 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9392 .await
9393 .pop()
9394 .expect("one job in, one result out");
9395 (seat, out)
9396}
9397
9398async fn wave(
9404 jobs: Vec<SeatJob>,
9405 sem: Arc<Semaphore>,
9406 ctx: &WaveCtx<'_>,
9407 state: &mut RunState,
9408 attempt: usize,
9409) -> Vec<(usize, SeatState, AgentOutcome)> {
9410 let WaveCtx {
9411 run,
9412 node,
9413 prompts,
9414 cache,
9415 round,
9416 carry_seats: _,
9417 pauses,
9418 } = *ctx;
9419 for job in &jobs {
9420 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9421 }
9422 if let Err(e) = state.save() {
9423 tracing::warn!("could not persist in-progress seats: {e:#}");
9428 }
9429 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9445 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9449 let cache_guard = if let Some(cache_dir) = cache {
9450 if jobs_had_a_writer {
9451 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9452 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9453 match acquire_cache_lease(
9454 state,
9455 cache_dir,
9456 &owner,
9457 resource_limit,
9458 node,
9459 node,
9460 &cancel,
9461 )
9462 .await
9463 {
9464 Ok(Some(g)) => Some(g),
9465 Ok(None) => {
9468 if let Some(block) = &mut state.resource_block {
9469 block.cause = crate::run::ResourceBlockCause::Cancelled {
9470 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9471 };
9472 }
9473 return block_wave(
9474 jobs,
9475 state,
9476 node,
9477 attempt,
9478 "parked while waiting for the shared build cache",
9479 );
9480 }
9481 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9484 Err(e) => {
9485 let why = format!("{e:#}");
9489 state.event(
9490 "resource",
9491 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9492 );
9493 return block_wave(jobs, state, node, attempt, &why);
9494 }
9495 }
9496 } else {
9497 None
9498 }
9499 } else {
9500 None
9501 };
9502 state.seats_spawning(node, attempt);
9505 if let Err(e) = state.save() {
9506 tracing::warn!("could not persist the start of a wave: {e:#}");
9507 }
9508 let slot_limit = resource_limit
9512 + jobs
9513 .iter()
9514 .map(|j| j.timeout)
9515 .max()
9516 .unwrap_or_default()
9517 .saturating_mul(jobs.len() as u32);
9518 let mut set = tokio::task::JoinSet::new();
9519 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9522 let overlay = prompts.overlay(node);
9523 let repository_wait = repository_wait(state, pauses);
9524 for (i, mut job) in jobs.into_iter().enumerate() {
9525 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9526 if cache.is_some() {
9527 job.prompt.push('\n');
9528 job.prompt
9529 .push_str(&prompt::build_cache_note(node, job.allow_write));
9530 }
9531 let sem = Arc::clone(&sem);
9532 let run = run.to_owned();
9533 let node = node.to_owned();
9534 let attachments = if node == "implement" {
9537 state.attachments.clone()
9538 } else {
9539 Vec::new()
9540 };
9541 let cache = cache
9552 .filter(|_| job.allow_write && cache_guard.is_some())
9553 .map(Path::to_path_buf);
9554 let spawned_tx = spawned_tx.clone();
9555 let repository_wait = repository_wait.clone();
9556 let coordination = state.config.coordination.clone();
9557 let fetched_base = state.fetched_base.clone();
9558 set.spawn(async move {
9559 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9563 Ok(p) => p,
9564 Err(_) => {
9565 return (
9566 i,
9567 job.seat,
9568 AgentOutcome::ResourceBlocked(format!(
9569 "no parallel slot became free within {}s",
9570 slot_limit.as_secs()
9571 )),
9572 false,
9573 );
9574 }
9575 };
9576 spawned_tx.send(job.seat.key.clone()).ok();
9577 let mut seat = job.seat;
9578 let out = repository_wait
9579 .scope(agent::invoke(
9580 &job.spec,
9581 &mut seat,
9582 &Invocation {
9583 scope: crate::agent::InvocationScope::Repository,
9584 coordination: coordination.as_ref(),
9585 fetched_base: fetched_base.as_ref(),
9586 cwd: &job.cwd,
9587 prompt: &job.prompt,
9588 timeout: job.timeout,
9589 allow_write: job.allow_write,
9590 unsandboxed: false,
9591 sessions: job.sessions,
9592 artifacts: &job.artifacts,
9593 stem: &job.stem,
9594 run: &run,
9595 node: &node,
9596 cache_dir: cache.as_deref(),
9597 attachments: &attachments,
9598 writable: &[],
9599 },
9600 ))
9601 .await;
9602 let interrupted = out
9603 .as_ref()
9604 .err()
9605 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9606 let out = match out {
9607 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9608 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9609 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9617 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9618 Ok(o) => AgentOutcome::Failed(format!(
9619 "process exited with no usable output ({})",
9620 crate::notices::exit_status(o.exit_code)
9621 )),
9622 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9623 AgentOutcome::ResourceBlocked(e.to_string())
9624 }
9625 Err(e) => AgentOutcome::Failed(e.to_string()),
9626 };
9627 (i, seat, out, interrupted)
9628 });
9629 }
9630 drop(spawned_tx);
9631 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9632 loop {
9633 let joined = tokio::select! {
9634 Some(key) = spawned_rx.recv() => {
9635 state.seat_spawned(&key);
9637 if let Err(e) = state.save() {
9638 tracing::warn!("could not persist a seat's start: {e:#}");
9639 }
9640 continue;
9641 }
9642 joined = set.join_next() => joined,
9643 };
9644 let Some(joined) = joined else {
9645 break;
9646 };
9647 let (i, seat, out, interrupted) = match joined {
9648 Ok(v) => v,
9649 Err(e) => {
9653 tracing::error!("agent task panicked: {e}");
9654 continue;
9655 }
9656 };
9657 state.seat_finished(&seat.key);
9658 if interrupted
9659 && node == "implement"
9660 && let Some(label) = state
9661 .candidates
9662 .iter()
9663 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9664 .map(|candidate| candidate.label)
9665 {
9666 state.candidate_preservation.insert(
9667 label,
9668 "interrupted implementation awaiting persistence".into(),
9669 );
9670 state.interrupted_implementations.insert(label);
9671 }
9672 if let AgentOutcome::ResourceBlocked(why) = &out
9673 && state.resource_block.is_none()
9674 {
9675 state.resource_block = Some(crate::run::ResourceBlock {
9676 cause: crate::run::ResourceBlockCause::Other,
9677 node: node.to_owned(),
9678 kind: if why.contains("disk pressure") {
9679 "disk pressure"
9680 } else if why.contains("repository coordination") {
9681 "repository coordination"
9682 } else {
9683 "parallel slot"
9684 }
9685 .to_owned(),
9686 detail: why.clone(),
9687 at: Timestamp::now(),
9688 });
9689 let activity = if interrupted {
9690 "agent execution interrupted"
9691 } else {
9692 "no agent was started"
9693 };
9694 state.event("resource", format!("{node}: {activity} - {why}"));
9695 }
9696 record_jobs(state, node, round, &seat.key, &out);
9697 if let Err(e) = state.save() {
9698 tracing::warn!("could not persist a seat's completion: {e:#}");
9699 }
9700 if collected.len() <= i {
9701 collected.resize_with(i + 1, || None);
9702 }
9703 collected[i] = Some((i, seat, out));
9704 }
9705 if state
9711 .active
9712 .values()
9713 .any(|a| a.node == node && a.attempt == attempt)
9714 {
9715 state
9716 .active
9717 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9718 if let Err(e) = state.save() {
9719 tracing::warn!("could not persist the end of a wave: {e:#}");
9720 }
9721 }
9722 if let Some(cache_dir) = cache
9729 && jobs_had_a_writer
9730 {
9731 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9732 }
9733 if let Some(guard) = cache_guard {
9734 guard.release();
9735 }
9736 collected.into_iter().flatten().collect()
9737}
9738
9739fn record_jobs(
9750 state: &mut RunState,
9751 node: &str,
9752 round: Option<usize>,
9753 seat: &str,
9754 out: &AgentOutcome,
9755) {
9756 let commands: &[agent::CommandEvidence] = match out {
9757 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9758 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9759 };
9760 let checked_at = Timestamp::now();
9761 for c in commands {
9762 state.jobs.push(JobRecord {
9763 node: node.to_owned(),
9764 round,
9765 seat: seat.to_owned(),
9766 id: c.id.clone(),
9767 description: c.description.clone(),
9768 checked_at,
9769 status: match c.exit_code {
9770 Some(0) => JobStatus::Completed,
9771 Some(_) => JobStatus::Failed,
9772 None => JobStatus::Unknown,
9773 },
9774 exit_code: c.exit_code,
9775 result_summary: c.result_summary.clone(),
9776 source: c.source.clone(),
9777 });
9778 }
9779}
9780
9781fn round_is_clean(
9802 blocking: usize,
9803 e2e_ok: bool,
9804 answered: usize,
9805 expected: usize,
9806 quota_missing: usize,
9807 policy: IncompleteReviewPolicy,
9808) -> bool {
9809 if blocking != 0 || !e2e_ok {
9810 return false;
9811 }
9812 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9813 return true;
9814 }
9815 answered > 0 && expected - answered <= quota_missing
9816}
9817
9818pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9842 if max_rounds == 0
9843 || reviews.last().is_some_and(|r| {
9844 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9845 })
9846 {
9847 return Some(RunStatus::Gating);
9848 }
9849 let last = reviews.last()?;
9850 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9851 if reviews.len() < max_rounds && !stagnant {
9852 return None;
9853 }
9854 if last.incomplete() && last.blocking == 0 {
9855 return Some(RunStatus::Blocked);
9856 }
9857 if last.e2e_status() == E2eStatus::ResourceBlocked
9858 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9859 {
9860 return None;
9861 }
9862 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9863 RunStatus::Gating
9864 } else {
9865 RunStatus::Blocked
9866 })
9867}
9868
9869pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9884 if nudged {
9885 (full / 4).max(Duration::from_secs(120)).min(full)
9886 } else {
9887 full
9888 }
9889}
9890
9891#[allow(clippy::too_many_arguments)]
9920async fn ask_json_wave<T>(
9921 jobs: Vec<SeatJob>,
9922 sem: Arc<Semaphore>,
9923 retries: usize,
9924 roster: &[AgentSpec],
9925 ctx: &WaveCtx<'_>,
9926 losses: &mut Vec<QuotaLoss>,
9927 state: &mut RunState,
9928 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9929) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9930where
9931 T: serde::de::DeserializeOwned + Send + 'static,
9932{
9933 ask_wave_with(
9934 jobs,
9935 sem,
9936 retries,
9937 roster,
9938 ctx,
9939 losses,
9940 state,
9941 &|text: &str| {
9942 let v = verdict::extract_json::<T>(text)?;
9943 validate(&v)?;
9944 Ok(v)
9945 },
9946 )
9947 .await
9948}
9949
9950#[allow(clippy::too_many_arguments)]
9959async fn ask_wave_with<T>(
9960 jobs: Vec<SeatJob>,
9961 sem: Arc<Semaphore>,
9962 retries: usize,
9963 roster: &[AgentSpec],
9964 ctx: &WaveCtx<'_>,
9965 losses: &mut Vec<QuotaLoss>,
9966 state: &mut RunState,
9967 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9968) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9969where
9970 T: Send + 'static,
9971{
9972 let n = jobs.len();
9973 let originals: Vec<SeatJob> = jobs;
9974 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9975 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9976 let mut nudges: Vec<usize> = vec![0; n];
9983 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9987 let mut tried: Vec<BTreeSet<String>> = specs
9988 .iter()
9989 .map(|s| BTreeSet::from([s.id.clone()]))
9990 .collect();
9991 let starts: Vec<usize> = specs
9992 .iter()
9993 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9994 .collect();
9995 let carry = ctx.carry_seats && !roster.is_empty();
9996 let carried: Vec<BTreeSet<String>> = originals
9999 .iter()
10000 .map(|j| {
10001 state
10002 .seat_history
10003 .get(&j.seat.key)
10004 .filter(|_| carry)
10005 .map(|h| h.failed.clone())
10006 .unwrap_or_default()
10007 })
10008 .collect();
10009 let mut prev: Vec<Option<FailClass>> = originals
10010 .iter()
10011 .map(|j| {
10012 state
10013 .seat_history
10014 .get(&j.seat.key)
10015 .filter(|_| carry)
10016 .and_then(|h| h.last_fail.clone())
10017 })
10018 .collect();
10019 let next_agent =
10020 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10021 let others: BTreeSet<String> = specs
10022 .iter()
10023 .enumerate()
10024 .filter(|(j, _)| *j != i)
10025 .map(|(_, s)| s.id.clone())
10026 .collect();
10027 pick_successor(
10028 roster,
10029 starts[i],
10030 tried,
10031 carry.then(|| &carried[i]),
10032 &others,
10033 )
10034 .cloned()
10035 };
10036 let mut fresh: Vec<Option<String>> = vec![None; n];
10037 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10038 let mut pending: Vec<usize> = (0..n).collect();
10039
10040 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10045 for round in 0..max_rounds {
10046 if pending.is_empty() {
10047 break;
10048 }
10049 let mut batch = Vec::with_capacity(pending.len());
10050 let mut renudged: Vec<&str> = Vec::new();
10051 for &i in &pending {
10052 let src = &originals[i];
10053 let full: &str = match &src.handover {
10058 Some(h) if specs[i].id != src.spec.id => h,
10059 _ => &src.prompt,
10060 };
10061 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10064 (
10065 full.to_owned(),
10066 src.timeout,
10067 format!("{}-{word}-{}", src.stem, specs[i].id),
10068 )
10069 } else if nudges[i] == 0 {
10070 (full.to_owned(), src.timeout, src.stem.clone())
10071 } else {
10072 renudged.push(src.seat.key.as_str());
10073 let why = done[i]
10074 .as_ref()
10075 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10076 .unwrap_or_else(|| "no parsable answer".to_owned());
10077 let nudge = prompt::nudge(&why);
10078 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10079 let prompt = if nudged {
10080 nudge
10081 } else {
10082 format!("{full}\n\n---\n\n{nudge}")
10083 };
10084 (
10085 prompt,
10086 retry_budget(src.timeout, nudged),
10087 format!("{}-retry{}", src.stem, nudges[i]),
10088 )
10089 };
10090 batch.push(SeatJob {
10091 spec: specs[i].clone(),
10092 seat: seats[i].clone(),
10093 cwd: src.cwd.clone(),
10094 prompt,
10095 timeout,
10096 allow_write: src.allow_write,
10097 sessions: src.sessions,
10098 artifacts: src.artifacts.clone(),
10099 stem,
10100 handover: None,
10101 });
10102 }
10103
10104 if !renudged.is_empty() {
10105 state.event(
10106 ctx.node,
10107 format!("retry {round}: re-asking {}", renudged.join(", ")),
10108 );
10109 }
10110 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10111 let mut still = Vec::new();
10112 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10113 seats[i] = seat;
10114 if let AgentOutcome::ResourceBlocked(why) = &out {
10117 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10118 continue;
10119 }
10120 let class = FailClass::of(&out);
10121 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10124 && nudges[i] < retries
10125 && !roster.is_empty();
10126 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10127 let next = should_hand_over(prev[i].as_ref(), &cur)
10128 .then(|| next_agent(i, &tried[i], &specs))
10129 .flatten();
10130 if carry {
10131 let h = state
10132 .seat_history
10133 .entry(originals[i].seat.key.clone())
10134 .or_default();
10135 h.failed.insert(specs[i].id.clone());
10136 h.last_fail = Some(cur.clone());
10137 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10138 h.last_ok = None;
10139 }
10140 if let Err(e) = state.save() {
10143 tracing::warn!("could not persist a seat's failure history: {e:#}");
10144 }
10145 }
10146 if let Some(next) = next {
10147 record_handover(
10148 state,
10149 ctx.node,
10150 &originals[i].seat.key,
10151 &specs[i].id,
10152 &next.id,
10153 &cur,
10154 &fail_reason(&out),
10155 );
10156 tried[i].insert(next.id.clone());
10157 prev[i] = Some(cur.clone());
10158 seats[i] =
10159 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10160 specs[i] = next;
10161 fresh[i] = Some(cur.stem_word().to_owned());
10162 nudges[i] = 0;
10163 done[i] = Some(Err(anyhow::anyhow!(
10164 "handed over after: {}",
10165 fail_reason(&out)
10166 )));
10167 still.push(i);
10168 continue;
10169 }
10170 }
10171 if carry
10172 && !nudge_first
10173 && let Some(cur) = class.clone()
10174 {
10175 let h = state
10177 .seat_history
10178 .entry(originals[i].seat.key.clone())
10179 .or_default();
10180 h.failed.insert(specs[i].id.clone());
10181 h.last_fail = Some(cur);
10182 }
10183 let parsed = match out {
10184 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10185 AgentOutcome::Quota(o) => {
10186 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10187 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10188 }
10189 AgentOutcome::Dropped(o) => {
10193 let why = o
10194 .dropped
10195 .as_ref()
10196 .map(|d| d.why.as_str())
10197 .unwrap_or("the CLI ended the stream without delivering its answer");
10198 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10199 }
10200 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10201 Err(anyhow::anyhow!(e))
10202 }
10203 };
10204 let quota = class == Some(FailClass::Quota);
10205 let failed = parsed.is_err();
10206 done[i] = Some(parsed);
10207 if carry && !failed {
10208 let h = state
10209 .seat_history
10210 .entry(originals[i].seat.key.clone())
10211 .or_default();
10212 h.failed.remove(&specs[i].id);
10213 h.last_ok = Some(specs[i].id.clone());
10214 h.last_fail = None;
10215 }
10216 let agent_failure = class.is_some()
10224 && !nudge_first
10225 && !roster.is_empty()
10226 && next_agent(i, &tried[i], &specs).is_some();
10227 if failed && !quota && !agent_failure && nudges[i] < retries {
10228 nudges[i] += 1;
10229 still.push(i);
10230 }
10231 }
10232 pending = still;
10233 }
10234
10235 for (i, q) in last_quota.into_iter().enumerate() {
10238 if let Some(reset) = q {
10239 losses.push(QuotaLoss {
10240 seat: originals[i].seat.key.clone(),
10241 node: ctx.node.to_owned(),
10242 at: Timestamp::now(),
10243 reset,
10244 });
10245 }
10246 }
10247
10248 seats
10249 .into_iter()
10250 .zip(done)
10251 .zip(nudges)
10252 .map(|((seat, res), attempts)| {
10253 (
10254 seat,
10255 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10256 attempts,
10257 )
10258 })
10259 .collect()
10260}
10261
10262fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10264 results
10265 .iter()
10266 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10267}
10268
10269fn block_wave(
10273 jobs: Vec<SeatJob>,
10274 state: &mut RunState,
10275 node: &str,
10276 attempt: usize,
10277 why: &str,
10278) -> Vec<(usize, SeatState, AgentOutcome)> {
10279 state.resource_wait = None;
10280 state
10281 .active
10282 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10283 if let Err(e) = state.save() {
10284 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10285 }
10286 jobs.into_iter()
10287 .enumerate()
10288 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10289 .collect()
10290}
10291
10292async fn acquire_cache_lease(
10305 state: &mut RunState,
10306 cache_dir: &Path,
10307 owner: &crate::cache::Owner,
10308 budget: Duration,
10309 node: &str,
10310 context: &str,
10311 cancel: &(dyn Fn() -> bool + Sync),
10312) -> Result<Option<crate::cache::Guard>> {
10313 let home = crate::run::home();
10314 let started = Instant::now();
10315 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10316 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10317 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10318 Err(e) => {
10319 state.event(
10320 "verify",
10321 format!("{context}: could not check the shared build cache: {e:#}"),
10322 );
10323 if let Err(e2) = state.save() {
10324 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10325 }
10326 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10327 }
10328 };
10329 state.event(
10330 "verify",
10331 format!(
10332 "{context}: waiting for the shared build cache at {} ({})",
10333 cache_dir.display(),
10334 busy.describe()
10335 ),
10336 );
10337 state.resource_wait = Some(crate::run::ResourceWait {
10340 node: context.to_owned(),
10341 kind: "shared build cache".to_owned(),
10342 holder: busy.describe(),
10343 since: Timestamp::now(),
10344 limit_secs: budget.as_secs(),
10345 });
10346 if let Err(e) = state.save() {
10347 tracing::warn!("could not persist a cache wait: {e:#}");
10348 }
10349 let remaining = budget.saturating_sub(started.elapsed());
10350 let waited = crate::cache::wait_for_unless(
10351 &home,
10352 cache_dir,
10353 owner,
10354 remaining,
10355 Duration::from_secs(5),
10356 cancel,
10357 )
10358 .await;
10359 state.resource_wait = None;
10360 let cause = match &waited {
10361 Ok(Some(_)) => None,
10362 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10363 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10364 Some(crate::run::ResourceBlockCause::Timeout {
10365 waited_secs: started.elapsed().as_secs(),
10366 limit_secs: budget.as_secs(),
10367 })
10368 }
10369 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10370 };
10371 if let Some(cause) = cause {
10372 let detail = match &waited {
10373 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10374 Err(error) => format!("{error:#}"),
10375 };
10376 state.resource_block = Some(crate::run::ResourceBlock {
10377 cause,
10378 node: node.to_owned(),
10379 kind: "shared build cache".into(),
10380 detail,
10381 at: Timestamp::now(),
10382 });
10383 state.event(
10384 "resource",
10385 state.resource_block.as_ref().unwrap().description(),
10386 );
10387 }
10388 if let Err(error) = state.save() {
10389 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10390 }
10391 waited
10392}
10393
10394#[derive(Debug)]
10399struct CacheCheckError(String);
10400
10401impl std::fmt::Display for CacheCheckError {
10402 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10403 f.write_str(&self.0)
10404 }
10405}
10406
10407impl std::error::Error for CacheCheckError {}
10408
10409fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10411 state
10412 .resource_block
10413 .get_or_insert_with(|| crate::run::ResourceBlock {
10414 cause: crate::run::ResourceBlockCause::Other,
10415 node: node.to_owned(),
10416 kind: "shared build cache".into(),
10417 detail: format!("{error:#}"),
10418 at: Timestamp::now(),
10419 });
10420 if let Err(error) = state.save() {
10421 tracing::warn!("could not persist cache resource block: {error:#}");
10422 }
10423}
10424
10425#[allow(clippy::too_many_arguments)]
10446async fn with_cache_lease<'s, F, Fut>(
10447 state: &'s mut RunState,
10448 cache_dir: Option<&Path>,
10449 node: &str,
10450 seat: &str,
10451 worktree: &Path,
10452 head: &str,
10453 budget: Duration,
10454 context: &str,
10455 wait: &crate::repo_lock::WaitPolicy,
10456 body: F,
10457) -> Result<(Vec<CommandOutcome>, bool)>
10458where
10459 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10460 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10461{
10462 let started = Instant::now();
10465 let home = crate::run::home();
10466 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10467 let guard = if let Some(cache_dir) = cache_dir {
10468 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10469 match acquire_cache_lease(
10470 state,
10471 cache_dir,
10472 &owner,
10473 resource_limit,
10474 node,
10475 context,
10476 &|| wait.cancelled(),
10477 )
10478 .await
10479 .and_then(|g| {
10480 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10481 }) {
10482 Ok(g) => Some(g),
10483 Err(e) => {
10484 record_cache_error(state, node, &e);
10485 return Ok((
10486 vec![CommandOutcome {
10487 command: "(waiting for the shared build cache)".into(),
10488 code: None,
10489 output_tail: e.to_string(),
10490 duration_ms: started.elapsed().as_millis() as u64,
10491 resource_blocked: true,
10492 }],
10493 false,
10494 ));
10495 }
10496 }
10497 } else {
10498 None
10499 };
10500 let _coordination = coordinate_operation(state, worktree, wait).await?;
10501 if let Some(cache_dir) = cache_dir {
10502 let identity = crate::cache::Identity::new(worktree, head);
10503 if let Err(e) = crate::cache::ensure_fresh(
10504 &home,
10505 cache_dir,
10506 &identity,
10507 &state.config.verify.cargo_manifests(),
10508 ) {
10509 record_cache_error(state, node, &e);
10510 state.event(
10511 "verify",
10512 format!(
10513 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10514 worktree.display(),
10515 short(head)
10516 ),
10517 );
10518 if let Some(guard) = guard {
10519 guard.release();
10520 }
10521 return Ok((
10522 vec![CommandOutcome {
10523 command: "(confirming the shared build cache is fresh)".into(),
10524 code: None,
10525 output_tail: e.to_string(),
10526 duration_ms: started.elapsed().as_millis() as u64,
10527 resource_blocked: true,
10528 }],
10529 false,
10530 ));
10531 }
10532 }
10533 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10534 if !timed_out_pids.is_empty() {
10535 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10536 }
10537 drop(_coordination);
10540 if let Some(guard) = guard {
10541 guard.release();
10542 }
10543 Ok((outcomes, retried))
10544}
10545
10546async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10558 wait_for_pids_with(
10559 pids,
10560 crate::proc::pid_alive,
10561 LEASE_RELEASE_POLL,
10562 LEASE_RELEASE_MAX_WAIT,
10563 )
10564 .await;
10565}
10566
10567async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10573 pids: &[u32],
10574 alive: F,
10575 poll: Duration,
10576 max_wait: Duration,
10577) {
10578 let deadline = Instant::now() + max_wait;
10579 loop {
10580 if pids.iter().all(|&pid| !alive(pid)) {
10581 return;
10582 }
10583 if Instant::now() >= deadline {
10584 return;
10585 }
10586 tokio::time::sleep(poll).await;
10587 }
10588}
10589
10590fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10597 outcomes.iter().any(|o| o.resource_blocked)
10598}
10599
10600pub(crate) enum GateFix {
10602 Retry,
10604 Stop,
10607 Defer,
10610}
10611
10612fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10620 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10621 red.peek().is_some()
10622 && red.all(|o| {
10623 !o.resource_blocked
10624 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10625 && !o.output_tail.trim().is_empty()
10626 })
10627}
10628
10629fn e2e_outcome_label(o: &CommandOutcome) -> String {
10633 if o.ok() {
10634 return "pass".to_owned();
10635 }
10636 let reason = if o.build_failed() {
10637 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10638 } else {
10639 format!("FAIL ({:?})", o.code)
10640 };
10641 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10642}
10643
10644async fn run_e2e_with_retry(
10652 state: &mut RunState,
10653 shell: &[String],
10654 commands: &[String],
10655 worktree: &Path,
10656 timeout: Duration,
10657 context: &str,
10658) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10659 let (mut e2e, mut timed_out_pids) = run_commands(
10660 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10661 )
10662 .await;
10663 for o in &e2e {
10664 state.event(
10665 "verify",
10666 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10667 );
10668 }
10669 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10673 if verify_retried {
10674 state.event(
10675 "verify",
10676 format!(
10677 "{context}: verify could not build/link, not a test result — retrying once \
10678 before concluding"
10679 ),
10680 );
10681 let retried = run_commands(
10682 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10683 )
10684 .await;
10685 e2e = retried.0;
10686 timed_out_pids.extend(retried.1);
10689 for o in &e2e {
10690 state.event(
10691 "verify",
10692 format!(
10693 "{context}: retry `{}` -> {}",
10694 o.command,
10695 e2e_outcome_label(o)
10696 ),
10697 );
10698 }
10699 }
10700 (e2e, verify_retried, timed_out_pids)
10701}
10702
10703#[allow(clippy::too_many_arguments)]
10719async fn run_commands(
10720 state: &mut RunState,
10721 node: &str,
10722 task: &str,
10723 attempt: usize,
10724 shell: &[String],
10725 commands: &[String],
10726 cwd: &Path,
10727 timeout: Duration,
10728) -> (Vec<CommandOutcome>, Vec<u32>) {
10729 if commands.is_empty() {
10730 return (Vec::new(), Vec::new());
10735 }
10736 let mut out = Vec::new();
10737 let mut timed_out_pids = Vec::new();
10738 let total = commands.len();
10739 for (idx, command) in commands.iter().enumerate() {
10740 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10741 if let Err(e) = state.save() {
10742 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10743 }
10744 let started = Instant::now();
10745 let mut cmd = tokio::process::Command::new(&shell[0]);
10746 cmd.quiet();
10747 cmd.args(&shell[1..])
10748 .arg(command)
10749 .current_dir(cwd)
10750 .stdin(std::process::Stdio::null())
10751 .stdout(std::process::Stdio::piped())
10752 .stderr(std::process::Stdio::piped())
10753 .kill_on_drop(true);
10754 let mut outputs = Vec::new();
10755 let protected = (|| -> Result<()> {
10756 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10757 outputs.push(guard);
10758 }
10759 if let Some(cache) = state.config.cache_dir() {
10760 outputs.push(crate::retention::OutputLease::cache(
10761 &crate::run::home(),
10762 &cache,
10763 false,
10764 )?);
10765 }
10766 Ok(())
10767 })();
10768 if let Err(e) = protected {
10769 out.push(CommandOutcome {
10770 command: command.clone(),
10771 code: None,
10772 output_tail: format!("output ownership unavailable: {e:#}"),
10773 duration_ms: 0,
10774 resource_blocked: true,
10775 });
10776 continue;
10777 }
10778 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10779 let (code, body) = match spawned {
10780 Ok(mut running) => {
10781 let pid = running.child().id();
10782 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10783 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10784 let mut paths = vec![cwd.to_owned()];
10785 if let Some(cache) = state.config.cache_dir() {
10786 paths.push(cache);
10787 }
10788 let mut pressure = None;
10789 let status = tokio::select! {
10790 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10791 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10792 pressure = Some(reason); None
10793 }
10794 };
10795 let (code, failure) = match status {
10796 Some(Ok(Ok(status))) => (
10797 status.code(),
10798 status
10799 .code()
10800 .is_none()
10801 .then(|| format!("command terminated: {status}")),
10802 ),
10803 Some(Ok(Err(error))) => {
10804 (None, Some(format!("failed waiting for command: {error}")))
10805 }
10806 Some(Err(_)) => {
10807 if !running.coordinated()
10811 && let Some(pid) = pid
10812 {
10813 timed_out_pids.push(pid);
10814 }
10815 (
10816 None,
10817 Some(format!("timed out after {}s", timeout.as_secs())),
10818 )
10819 }
10820 None => (None, None),
10821 };
10822 running.finish().await;
10823 if let Some(reason) = pressure {
10824 state.resource_block = Some(crate::run::ResourceBlock {
10825 cause: crate::run::ResourceBlockCause::Other,
10826 node: node.to_owned(),
10827 kind: "disk pressure".to_owned(),
10828 detail: reason.clone(),
10829 at: Timestamp::now(),
10830 });
10831 out.push(CommandOutcome {
10832 command: command.clone(),
10833 code: None,
10834 output_tail: reason,
10835 duration_ms: started.elapsed().as_millis() as u64,
10836 resource_blocked: true,
10837 });
10838 break;
10839 }
10840 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10841 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10842 if let Some(failure) = failure {
10843 body.push('\n');
10844 body.push_str(&failure);
10845 }
10846 (code, body)
10847 }
10848 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10849 };
10850 out.push(CommandOutcome {
10851 command: command.clone(),
10852 code,
10853 output_tail: tail(&body, OUTPUT_TAIL),
10854 duration_ms: started.elapsed().as_millis() as u64,
10855 resource_blocked: false,
10856 });
10857 }
10858 state.task_finished(task);
10859 if let Err(e) = state.save() {
10860 tracing::warn!("could not persist the end of {task}: {e:#}");
10861 }
10862 (out, timed_out_pids)
10863}
10864
10865fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10877 let repo = repo.display();
10878 match style {
10879 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10880 MergeStyle::Squash => {
10881 let subject = message
10884 .lines()
10885 .next()
10886 .unwrap_or(branch)
10887 .replace(['\\', '"', '$', '`'], "");
10888 format!(
10889 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10890 )
10891 }
10892 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10893 }
10894}
10895
10896const PR_TITLE_MAX: usize = 240;
10907
10908fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10919 let line = queue::first_line(text)?;
10920 let chars: Vec<char> = line.chars().collect();
10921 let (kept, mark) = if chars.len() <= max {
10922 (line.to_owned(), "")
10923 } else if let Some(end) = sentence_end(&chars, max) {
10924 (chars[..end].iter().collect::<String>(), "")
10925 } else {
10926 let room = max.saturating_sub(3);
10927 let cut = if chars[room].is_whitespace() {
10930 room
10931 } else {
10932 chars[..room]
10933 .iter()
10934 .rposition(|c| c.is_whitespace())
10935 .unwrap_or(room)
10936 };
10937 let head: String = chars[..cut].iter().collect();
10938 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10939 (head.to_owned(), "...")
10940 };
10941 if kept.is_empty() {
10942 return None;
10943 }
10944 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10945 return None;
10946 }
10947 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10948 return None;
10949 }
10950 Some(format!("{kept}{mark}"))
10951}
10952
10953fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10957 const MIN: usize = 20;
10958 for i in MIN..max.min(chars.len()) {
10959 if !matches!(chars[i], '.' | '!' | '?') {
10960 continue;
10961 }
10962 let Some(&next) = chars.get(i + 1) else {
10963 continue;
10964 };
10965 if !next.is_whitespace() {
10966 continue;
10967 }
10968 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10969 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10970 continue;
10971 }
10972 let word: String = chars[..i]
10973 .iter()
10974 .rev()
10975 .take_while(|c| !c.is_whitespace())
10976 .collect::<Vec<_>>()
10977 .into_iter()
10978 .rev()
10979 .collect();
10980 let word = word.to_ascii_lowercase();
10981 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10982 continue;
10983 }
10984 let end = chars[..i]
10985 .iter()
10986 .rposition(|c| !c.is_whitespace())
10987 .map_or(i, |p| p + 1);
10988 return Some(end);
10989 }
10990 None
10991}
10992
10993struct PrMessage {
10998 title: String,
10999 body: String,
11000}
11001
11002impl PrMessage {
11003 fn commit_message(&self) -> String {
11007 format!("{}\n\n{}", self.title, self.body)
11008 }
11009}
11010
11011fn title_marker(line: &str) -> Option<&str> {
11013 let line = line.trim();
11014 let head = line.get(..6)?;
11015 head.eq_ignore_ascii_case("title:")
11016 .then(|| line[6..].trim())
11017}
11018
11019fn summary_title(summary: &str) -> Option<String> {
11024 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11025 let raw = title_marker(first)?;
11026 if raw.is_empty() {
11027 return None;
11028 }
11029 let title = queue::title_from(raw, PR_TITLE_MAX);
11030 let lower = title.to_ascii_lowercase();
11031 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11032 return None;
11033 }
11034 Some(title)
11035}
11036
11037const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11039
11040pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11042
11043pub(crate) fn is_review_run(state: &RunState) -> bool {
11047 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11048}
11049
11050fn review_title(state: &RunState) -> Option<String> {
11055 english_subject(
11056 state.reviewed_commits.as_ref()?.first()?,
11057 state.config.graph.github_english(),
11058 )
11059}
11060
11061fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11064 let raw = raw.trim();
11065 if raw.is_empty() {
11066 return None;
11067 }
11068 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11069 return None;
11070 }
11071 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11072 return None;
11073 }
11074 let title = queue::title_from(raw, PR_TITLE_MAX);
11075 let lower = title.to_ascii_lowercase();
11076 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11077 return None;
11078 }
11079 Some(title)
11080}
11081
11082#[derive(Debug, Clone, PartialEq, Eq)]
11085struct BranchFacts {
11086 commits: Vec<(String, String)>,
11088 stat: String,
11090}
11091
11092const STAT_FILE_LINES: usize = 25;
11094const COMMITS_MAX_CHARS: usize = 20_000;
11096
11097async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11100 let commits = git::commit_log(repo, base, branch).await.ok()?;
11101 if commits.is_empty() {
11102 return None;
11103 }
11104 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11105 let lines: Vec<&str> = stat.lines().collect();
11106 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11107 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11108 let more = format!(" ... {omitted} more file(s)");
11109 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11110 kept.push(&more);
11111 kept.push(lines[lines.len() - 1]);
11112 kept.join("\n")
11113 } else {
11114 lines.join("\n")
11115 };
11116 Some(BranchFacts { commits, stat })
11117}
11118
11119fn markdown_safe(text: &str) -> String {
11122 text.replace("</details>", "</details>")
11123 .replace("\x60\x60\x60", "~~~")
11124}
11125
11126fn neutral_title(state: &RunState, winner: char) -> String {
11127 format!(
11128 "chore: land candidate {} of run {}",
11129 winner.to_ascii_uppercase(),
11130 state.id
11131 )
11132}
11133
11134pub(crate) async fn repair_landing_gate(
11137 state: &mut RunState,
11138 winner: &Candidate,
11139 outcomes: &[CommandOutcome],
11140) -> Result<GateFix> {
11141 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11142 return Ok(GateFix::Stop);
11143 }
11144 let mut runner = Runner {
11145 roles: state.config.resolve_roles()?,
11146 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11147 state: state.clone(),
11148 pause: Pause::new(),
11149 interrupt: Pause::new(),
11150 };
11151 let result = runner.gate_fix_round(winner, outcomes).await;
11152 *state = runner.state;
11153 result
11154}
11155
11156pub(crate) async fn check_gate_repair_e2e(
11159 state: &mut RunState,
11160 winner: &Candidate,
11161 wait: &crate::repo_lock::WaitPolicy,
11162) -> Result<GateFix> {
11163 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11164 return Ok(GateFix::Retry);
11165 }
11166 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11167 if state
11168 .gate_repair_e2e
11169 .as_ref()
11170 .is_some_and(|r| r.head == head && r.passed)
11171 {
11172 return Ok(GateFix::Retry);
11173 }
11174 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11175 head: head.clone(),
11176 passed: false,
11177 outcomes: Vec::new(),
11178 });
11179 state.save()?;
11180 let commands = state.config.verify.e2e.clone();
11181 let outcomes = if commands.is_empty() {
11182 Vec::new()
11183 } else {
11184 let shell = state.config.shell();
11185 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11186 let cache_dir = state.config.cache_dir();
11187 let (outcomes, _) = with_cache_lease(
11188 state,
11189 cache_dir.as_deref(),
11190 "e2e",
11191 "e2e",
11192 &winner.worktree,
11193 &head,
11194 timeout,
11195 "gate repair e2e",
11196 wait,
11197 |state, budget| async move {
11198 run_e2e_with_retry(
11199 state,
11200 &shell,
11201 &commands,
11202 &winner.worktree,
11203 budget,
11204 "gate repair e2e",
11205 )
11206 .await
11207 },
11208 )
11209 .await?;
11210 outcomes
11211 };
11212 let deferred = verify_inconclusive(&outcomes);
11213 let passed = outcomes.iter().all(CommandOutcome::ok)
11214 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11215 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11216 head,
11217 passed,
11218 outcomes,
11219 });
11220 state.save()?;
11221 if deferred {
11222 Ok(GateFix::Defer)
11223 } else if !passed {
11224 state.event(
11225 "gate",
11226 "verify.e2e failed after gate repair; nothing published",
11227 );
11228 Ok(GateFix::Stop)
11229 } else {
11230 Ok(GateFix::Retry)
11231 }
11232}
11233
11234pub(crate) async fn run_landing_gate(
11236 state: &mut RunState,
11237 winner: &Candidate,
11238 wait: &crate::repo_lock::WaitPolicy,
11239) -> Result<Vec<CommandOutcome>> {
11240 let shell = state.config.shell();
11241 let gate_commands = state.config.verify.gate.clone();
11242 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11243 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11244 Vec::new()
11245 } else {
11246 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11247 let cache_dir = state.config.cache_dir();
11248 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11249 let (outcomes, _) = with_cache_lease(
11250 state,
11251 cache_dir.as_deref(),
11252 "gate",
11253 "gate",
11254 &winner.worktree,
11255 &head,
11256 timeout,
11257 "final gate",
11258 wait,
11259 |state, budget| {
11260 let shell = shell.clone();
11261 let gate_commands = gate_commands.clone();
11262 let winner = winner.clone();
11263 async move {
11264 Runner::run_pre_gate(state, &winner).await;
11265 let (outcomes, timed_out_pids) = run_commands(
11266 state,
11267 "gate",
11268 "gate",
11269 0,
11270 &shell,
11271 &gate_commands,
11272 &winner.worktree,
11273 budget,
11274 )
11275 .await;
11276 (outcomes, false, timed_out_pids)
11277 }
11278 },
11279 )
11280 .await?;
11281 outcomes
11282 };
11283 if outcomes.is_empty() {
11284 state.event("gate", "no gate commands configured; not tested");
11289 }
11290 for o in &outcomes {
11291 state.event(
11292 "gate",
11293 format!(
11294 "`{}` -> {}",
11295 o.command,
11296 if o.ok() {
11297 "pass".to_owned()
11298 } else {
11299 format!(
11300 "FAIL ({:?})\n{}",
11301 o.code,
11302 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11303 )
11304 }
11305 ),
11306 );
11307 }
11308 Ok(outcomes)
11309}
11310
11311pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11317 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11318 ""
11319 } else {
11320 pr_title
11321 }
11322}
11323
11324pub fn landing_subject_source(state: &RunState) -> String {
11328 if is_review_run(state) {
11329 let winner = state.candidates.first().map_or('A', |c| c.label);
11330 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11331 }
11332 state.instruction.clone()
11333}
11334
11335fn summary_without_title(summary: &str) -> String {
11338 let mut lines = summary.trim().lines().peekable();
11339 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11340 lines.next();
11341 }
11342 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11343}
11344
11345#[cfg(test)]
11357fn pr_message(state: &RunState, winner: char) -> PrMessage {
11358 pr_message_with(state, winner, None)
11359}
11360
11361#[cfg(test)]
11364fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11365 let raw = pr_message_raw(
11366 state,
11367 winner,
11368 facts,
11369 crate::pr_outcome::FinalHead::Unreported,
11370 ChangeNote::NotAsked,
11371 );
11372 let id = crate::scrub::Identity::current();
11373 PrMessage {
11374 title: crate::scrub::scrub(&raw.title, &id),
11375 body: crate::scrub::scrub(&raw.body, &id),
11376 }
11377}
11378
11379#[derive(Debug, Clone, Copy)]
11381enum ChangeNote<'a> {
11382 NotAsked,
11384 Summary(&'a crate::run::ChangeSummary),
11386 Unavailable(&'a str),
11388}
11389
11390fn pr_message_raw(
11391 state: &RunState,
11392 winner: char,
11393 facts: Option<&BranchFacts>,
11394 head: crate::pr_outcome::FinalHead<'_>,
11395 change: ChangeNote<'_>,
11396) -> PrMessage {
11397 if !matches!(change, ChangeNote::NotAsked) {
11398 let (title, mut body) = match change {
11399 ChangeNote::Summary(c) => (
11400 c.title.clone(),
11401 format!("## Summary\n\n{}\n\n", c.body.trim()),
11402 ),
11403 ChangeNote::Unavailable(why) => (
11404 neutral_title(state, winner),
11405 format!(
11406 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11407 ),
11408 ),
11409 ChangeNote::NotAsked => unreachable!(),
11410 };
11411 body.push_str(&public_pr_evidence(state, head));
11412 body.push_str(&format!(
11413 "\n---\nmagi:run/{} magi:candidate-{}\n",
11414 state.id,
11415 winner.to_ascii_lowercase()
11416 ));
11417 return PrMessage { title, body };
11418 }
11419 pr_preview(state, winner, facts, head)
11420}
11421
11422fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11425 let mut body = crate::pr_outcome::review_rounds(state);
11426 body.push_str(&crate::pr_outcome::final_verification(state, head));
11427 let open = state.last_round_findings();
11428 if !open.is_empty() {
11429 body.push_str("## Open review findings\n\n");
11430 for f in open {
11431 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11432 }
11433 body.push('\n');
11434 }
11435 if let Some(c) = state.change_summary.as_ref()
11436 && head == crate::pr_outcome::FinalHead::At(&c.head)
11437 {
11438 body.push_str(&format!(
11439 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11440 c.base, c.head
11441 ));
11442 }
11443 body
11444}
11445
11446fn pr_preview(
11448 state: &RunState,
11449 winner: char,
11450 facts: Option<&BranchFacts>,
11451 head: crate::pr_outcome::FinalHead<'_>,
11452) -> PrMessage {
11453 let summary = state
11454 .candidates
11455 .iter()
11456 .find(|c| c.label == winner)
11457 .map(|c| c.summary.as_str())
11458 .unwrap_or_default();
11459 let review = is_review_run(state);
11462 let title = if review {
11463 facts
11464 .and_then(|f| {
11465 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11466 })
11467 .or_else(|| review_title(state))
11468 .or_else(|| {
11469 state
11470 .candidates
11471 .iter()
11472 .find(|c| c.label == winner)
11473 .filter(|c| !c.branch.starts_with("magi/"))
11474 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11475 })
11476 .unwrap_or_else(|| neutral_title(state, winner))
11477 } else {
11478 summary_title(summary).unwrap_or_else(|| {
11479 english_title(
11480 &state.instruction,
11481 PR_TITLE_MAX,
11482 state.config.graph.github_english(),
11483 )
11484 .unwrap_or_else(|| neutral_title(state, winner))
11485 })
11486 };
11487
11488 let mut body = String::new();
11489 let what = summary_without_title(summary);
11490 if !what.is_empty() {
11491 body.push_str("## Summary\n\n");
11492 body.push_str(&what);
11493 body.push_str("\n\n");
11494 }
11495
11496 body.push_str(&crate::pr_outcome::review_rounds(state));
11498 body.push_str(&crate::pr_outcome::final_verification(state, head));
11499 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11500
11501 let open = state.open_findings();
11502 if !open.is_empty() {
11503 body.push_str("## Open review findings\n\n");
11504 for f in &open {
11505 body.push_str(&format!(
11506 "- `{}` [{:?}] {}\n",
11507 f.id, f.severity, "Context retained in local review records."
11508 ));
11509 }
11510 body.push('\n');
11511 }
11512
11513 if let Some(fix) = fix
11514 && !fix.rejected.is_empty()
11515 {
11516 body.push_str("## Declined by the fixer\n\n");
11517 for r in &fix.rejected {
11518 body.push_str(&format!(
11519 "- `{}`: {}\n",
11520 r.id, "Reason retained in local review records."
11521 ));
11522 }
11523 body.push('\n');
11524 }
11525
11526 if review {
11527 body.push_str("## Commits under review\n\n");
11529 if let Some(facts) = facts {
11530 let mut left = COMMITS_MAX_CHARS;
11531 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11532 let mut entry = format!("- {}\n", markdown_safe(subject));
11533 for l in markdown_safe(text).lines() {
11534 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11535 }
11536 if left == 0 {
11537 body.push_str(&format!(
11538 "- ... {} more commit(s)\n",
11539 facts.commits.len() - i
11540 ));
11541 break;
11542 }
11543 if entry.len() > left {
11544 let mut end = left;
11547 while !entry.is_char_boundary(end) {
11548 end -= 1;
11549 }
11550 entry.truncate(end);
11551 entry.push_str("\n ... (truncated)\n");
11552 left = 0;
11553 } else {
11554 left -= entry.len();
11555 }
11556 body.push_str(&entry);
11557 }
11558 if !facts.stat.trim().is_empty() {
11559 body.push_str(&format!(
11560 "\n## Diff stat\n\n```\n{}\n```\n",
11561 markdown_safe(facts.stat.trim())
11562 ));
11563 }
11564 } else {
11565 match &state.reviewed_commits {
11566 Some(subjects) => {
11567 for s in subjects {
11568 body.push_str(&format!("- {}\n", s.trim()));
11569 }
11570 }
11571 None => {
11572 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11575 body.push_str(rest.trim());
11576 body.push('\n');
11577 }
11578 }
11579 }
11580 } else {
11581 let task = state.instruction.trim();
11582 let task = if task.is_empty() {
11583 "(empty task)"
11584 } else {
11585 task
11586 };
11587 body.push_str(&format!(
11588 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11589 task.replace("</details>", "</details>")
11590 ));
11591 }
11592
11593 body.push_str(&format!(
11594 "\n---\nmagi:run/{} magi:candidate-{}\n",
11595 state.id,
11596 winner.to_ascii_lowercase()
11597 ));
11598
11599 PrMessage { title, body }
11600}
11601
11602fn seeded_instruction(state: &RunState) -> String {
11605 let mut inputs = Vec::new();
11606 if let Some(start) = &state.sources.start_from {
11607 inputs.push(format!(
11608 "Start from `{}` at `{}` (including ancestry).",
11609 start.token, start.sha
11610 ));
11611 }
11612 for pick in &state.sources.pick_commits {
11613 inputs.push(format!(
11614 "Pick `{}` at `{}` (single commit).",
11615 pick.token, pick.sha
11616 ));
11617 }
11618 if inputs.is_empty() {
11619 state.instruction.clone()
11620 } else {
11621 format!(
11622 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11623 state.instruction,
11624 inputs.join("\n")
11625 )
11626 }
11627}
11628
11629async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11633 let base = &state.base_branch;
11634 let mut against = base.clone();
11635 if mode == MergeMode::Pr {
11636 let remote = &state.config.merge.remote;
11640 let tracking = format!("{remote}/{base}");
11641 let fetched = git::fetch(repo, remote, base).await;
11642 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11643 against = tracking;
11644 } else {
11645 return false;
11646 }
11647 }
11648 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11649}
11650
11651fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11654 let mut detail = format!(
11655 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11656 nothing to open a pull request for"
11657 );
11658 detail.push_str(
11659 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11660 );
11661 detail
11662}
11663
11664#[derive(Debug, PartialEq, Eq)]
11667enum PrPlan {
11668 Create,
11669 Adopt { url: String, title: String },
11670 Stop(String),
11671}
11672
11673fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11676 match found {
11677 Ok(land::OpenPr::None) => PrPlan::Create,
11678 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11679 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11680 "several open pull requests exist for this branch, not picking one: {}",
11681 urls.join(" ")
11682 )),
11683 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11684 }
11685}
11686
11687pub(crate) async fn gh_pr_create(
11689 cwd: &Path,
11690 base: &str,
11691 head: &str,
11692 title: &str,
11693 body: &str,
11694) -> Result<String> {
11695 let out = tokio::process::Command::new("gh")
11696 .args([
11697 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11698 ])
11699 .current_dir(cwd)
11700 .quiet()
11701 .stdin(std::process::Stdio::null())
11702 .output()
11703 .await
11704 .context("spawn gh")?;
11705 if out.status.success() {
11706 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11707 } else {
11708 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11709 }
11710}
11711
11712async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11714 match std::fs::symlink_metadata(path) {
11715 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11716 git::worktree_remove_clean(repo, path).await?;
11719 return Ok(());
11720 }
11721 Err(e) => return Err(e.into()),
11722 Ok(_) => {}
11723 }
11724 if !git::is_clean(path).await? {
11725 bail!(
11726 "worktree has uncommitted work; refusing removal: {}",
11727 path.display()
11728 );
11729 }
11730 if !git::worktree_remove_clean(repo, path).await? {
11731 bail!("worktree removal refused: {}", path.display());
11732 }
11733 Ok(())
11734}
11735
11736pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11745 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11746 bail!("refusing fold: {reason}");
11747 }
11748 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11749 bail!("candidate preservation incomplete; refusing cleanup");
11750 }
11751 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11755 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11756 let _driver = crate::driver::maintenance(home, &state.id)?;
11757 let mut output_guards = Vec::new();
11758 for entry in std::fs::read_dir(state.worktree_root())
11759 .into_iter()
11760 .flatten()
11761 .flatten()
11762 {
11763 if entry.path().join(".git").is_file() {
11764 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11765 }
11766 }
11767 let repo = state.repo.clone();
11768 let wait = repository_wait(state, None);
11771 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11772 Ok(lease) => lease,
11773 Err(e) => {
11774 state.save_under(home)?;
11775 return Err(e);
11776 }
11777 };
11778 let _source_outputs = if state.status == RunStatus::Merged
11781 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11782 {
11783 verify_merged_fold_sources(state, drop_winner, home).await?
11784 } else {
11785 Vec::new()
11786 };
11787 let root = state.worktree_root();
11788 let winner = state.tally.as_ref().map(|t| t.winner);
11789 let mut removed = Vec::new();
11790
11791 for i in 0..state.candidates.len() {
11792 let c = state.candidates[i].clone();
11793 let is_winner = Some(c.label) == winner;
11794 if is_winner && !drop_winner {
11795 continue;
11796 }
11797 let existed = c.worktree.exists();
11798 remove_preserved_worktree(&repo, &c.worktree).await?;
11801 if existed {
11802 removed.push(c.worktree.to_string_lossy().into_owned());
11803 }
11804 let handed_over = state.released_branches.contains(&c.branch);
11807 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11808 if !git::branch_delete(&repo, &c.branch).await? {
11809 bail!("candidate branch removal refused");
11810 }
11811 removed.push(c.branch.clone());
11812 }
11813 state.candidates[i].folded = true;
11814 }
11815
11816 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11817 let path = name.path();
11818 let keep = !drop_winner
11819 && winner.is_some_and(|w| {
11820 path.file_name()
11821 .is_some_and(|n| n == format!("cand-{w}").as_str())
11822 });
11823 if keep {
11824 continue;
11825 }
11826 remove_preserved_worktree(&repo, &path).await?;
11827 removed.push(path.to_string_lossy().into_owned());
11828 }
11829
11830 remove_if_empty(&root);
11839
11840 if state.enabled_worktree_config && drop_winner {
11841 git::release_worktree_config(&repo).await.ok();
11845 state.enabled_worktree_config = false;
11846 }
11847 state.save_under(home)?;
11848 Ok(removed)
11849}
11850
11851async fn verify_merged_fold_sources(
11852 state: &RunState,
11853 drop_winner: bool,
11854 home: &Path,
11855) -> Result<Vec<std::fs::File>> {
11856 let winner = state.tally.as_ref().map(|t| t.winner);
11857 let mut heads = BTreeSet::new();
11858 for candidate in &state.candidates {
11859 if Some(candidate.label) == winner && !drop_winner {
11860 continue;
11861 }
11862 anyhow::ensure!(
11863 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11864 "candidate recovery source is outside the inventoried run root: {}",
11865 candidate.worktree.display()
11866 );
11867 match std::fs::symlink_metadata(&candidate.worktree) {
11868 Ok(_) => {}
11869 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11870 Err(error) => return Err(error.into()),
11871 }
11872 if !state.released_branches.contains(&candidate.branch)
11873 && git::branch_exists(&state.repo, &candidate.branch).await?
11874 {
11875 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11876 }
11877 }
11878 let mut cargo_guards = Vec::new();
11879 let entries = match std::fs::read_dir(state.worktree_root()) {
11880 Ok(entries) => Some(entries),
11881 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11882 Err(error) => return Err(error.into()),
11883 };
11884 for entry in entries.into_iter().flatten() {
11885 let path = entry?.path();
11886 let metadata = std::fs::symlink_metadata(&path)?;
11887 anyhow::ensure!(
11888 metadata.is_dir() && !metadata.file_type().is_symlink(),
11889 "non-directory or symlink recovery source retained: {}",
11890 path.display()
11891 );
11892 if !drop_winner
11893 && winner.is_some_and(|w| {
11894 path.file_name()
11895 .is_some_and(|name| name == format!("cand-{w}").as_str())
11896 })
11897 {
11898 continue;
11899 }
11900 anyhow::ensure!(
11901 path.join(".git").is_file(),
11902 "unregistered recovery source retained: {}",
11903 path.display()
11904 );
11905 anyhow::ensure!(
11906 git::git(
11907 &path,
11908 &[
11909 "status",
11910 "--porcelain",
11911 "--untracked-files=all",
11912 "--ignored",
11913 "--",
11914 ".",
11915 ":(exclude)target"
11916 ]
11917 )
11918 .await?
11919 .is_empty(),
11920 "source changes or ignored recovery files retained: {}",
11921 path.display()
11922 );
11923 match std::fs::symlink_metadata(path.join("target")) {
11924 Ok(_) => {
11925 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11926 }
11927 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11928 Err(error) => return Err(error.into()),
11929 }
11930 for checkpoint in [
11931 "rebase-merge",
11932 "rebase-apply",
11933 "sequencer",
11934 "CHERRY_PICK_HEAD",
11935 "MERGE_HEAD",
11936 "REBASE_HEAD",
11937 "REVERT_HEAD",
11938 "BISECT_LOG",
11939 "AUTO_MERGE",
11940 "MERGE_MSG",
11941 ] {
11942 let location = git::git(
11943 &path,
11944 &[
11945 "rev-parse",
11946 "--path-format=absolute",
11947 "--git-path",
11948 checkpoint,
11949 ],
11950 )
11951 .await?;
11952 anyhow::ensure!(
11953 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11954 if e.kind() == std::io::ErrorKind::NotFound),
11955 "Git recovery checkpoint retained: {checkpoint}"
11956 );
11957 }
11958 heads.insert(git::rev_parse(&path, "HEAD").await?);
11959 }
11960 crate::fold_evidence::preserve(state, home, &heads).await?;
11961
11962 Ok(cargo_guards)
11963}
11964
11965fn remove_if_empty(dir: &Path) {
11976 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11977 std::fs::remove_dir(dir).ok();
11978 }
11979}
11980
11981pub fn worst_open(state: &RunState) -> Option<Severity> {
11983 state
11984 .reviews
11985 .last()?
11986 .reviews
11987 .iter()
11988 .flat_map(|r| r.findings.iter())
11989 .map(|f| f.severity)
11990 .max()
11991}
11992
11993async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11997 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11998 git::acquire_worktree_config(repo).await?;
11999 state.enabled_worktree_config = true;
12000 }
12001 Ok(())
12002}
12003
12004#[cfg(test)]
12005mod tests {
12006 #[tokio::test]
12007 async fn launcher_base_fetch_requires_remote_success() {
12008 let tmp = tempfile::tempdir().unwrap();
12009 let repo = tmp.path().join("repo");
12010 let remote = tmp.path().join("remote");
12011 std::fs::create_dir(&repo).unwrap();
12012 std::fs::create_dir(&remote).unwrap();
12013 git::git(&remote, &["init", "--bare", "-b", "main"])
12014 .await
12015 .unwrap();
12016 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12017 git::git(
12018 &repo,
12019 &[
12020 "-c",
12021 "user.name=Fixture",
12022 "-c",
12023 "user.email=fixture@example.invalid",
12024 "commit",
12025 "--allow-empty",
12026 "-m",
12027 "fixture",
12028 ],
12029 )
12030 .await
12031 .unwrap();
12032 git::git(
12033 &repo,
12034 &["remote", "add", "origin", remote.to_str().unwrap()],
12035 )
12036 .await
12037 .unwrap();
12038 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12039 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12040 assert_eq!(
12041 resolve_base(&repo, "main", "origin", None)
12042 .await
12043 .unwrap()
12044 .commit,
12045 head
12046 );
12047 #[cfg(unix)]
12048 {
12049 let policy: crate::config::Coordination = toml::from_str(
12050 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12051 ).unwrap();
12052 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12053 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12054 let held = crate::repo_lock::SeatLease::operation_with(
12055 &repo,
12056 Some(&policy),
12057 &crate::repo_lock::WaitPolicy::default(),
12058 )
12059 .await
12060 .unwrap();
12061 assert!(held.is_some());
12062 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12063 tokio::pin!(fetch);
12064 assert!(
12065 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12066 .await
12067 .is_err()
12068 );
12069 assert!(
12070 !repo.join(".git/FETCH_HEAD").exists(),
12071 "no fetch before admission"
12072 );
12073 drop(held);
12074 let fetched = fetch.await.unwrap();
12075 assert_eq!(fetched.commit, head);
12076 assert_eq!(fetched.remote, "origin");
12077 assert_eq!(fetched.branch, "main");
12078 }
12079 git::git(
12081 &repo,
12082 &[
12083 "remote",
12084 "set-url",
12085 "origin",
12086 tmp.path().join("missing").to_str().unwrap(),
12087 ],
12088 )
12089 .await
12090 .unwrap();
12091 let error = resolve_base(&repo, "main", "origin", None)
12092 .await
12093 .unwrap_err()
12094 .to_string();
12095 assert!(error.contains("cannot read origin/main"));
12096 assert!(error.contains("refusing to branch off"));
12097 }
12098
12099 #[test]
12100 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12101 let leaked = vec!["chore(deps): update a crate".to_owned()];
12102 let own = "fix(daemon): apply a chosen action";
12103 assert!(should_retitle("", own, &leaked));
12104 assert!(should_retitle(
12105 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12106 own,
12107 &leaked
12108 ));
12109 assert!(should_retitle(
12110 "chore: land candidate A of run 1",
12111 own,
12112 &leaked
12113 ));
12114 assert!(should_retitle(
12115 "magi: candidate A (uncommitted work)",
12116 own,
12117 &leaked
12118 ));
12119 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12120 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12121 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12122 }
12123
12124 #[test]
12125 fn pr_merge_plan_creates_adopts_or_stops() {
12126 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12127 assert_eq!(
12128 pr_merge_plan(Ok(land::OpenPr::One {
12129 url: "u".into(),
12130 title: "t".into()
12131 })),
12132 PrPlan::Adopt {
12133 url: "u".into(),
12134 title: "t".into()
12135 }
12136 );
12137 let PrPlan::Stop(many) =
12138 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12139 else {
12140 panic!("many must stop");
12141 };
12142 assert!(many.contains('a') && many.contains('b'));
12143 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12144 panic!("a failed lookup must stop");
12145 };
12146 assert!(err.contains("bad token"));
12147 }
12148
12149 use super::*;
12150 use crate::run::GateStatus;
12151 use std::collections::BTreeMap;
12152 use std::time::Duration;
12153
12154 fn conductor() -> AgentSpec {
12155 AgentSpec {
12156 id: "conductor".to_owned(),
12157 kind: crate::config::AgentKind::Command,
12158 model: None,
12159 command: vec!["true".to_owned()],
12160 extra_args: Vec::new(),
12161 env: BTreeMap::new(),
12162 prompt_delivery: None,
12163 max_turn_secs: None,
12164 }
12165 }
12166
12167 fn spec(id: &str) -> AgentSpec {
12168 AgentSpec {
12169 id: id.to_owned(),
12170 kind: crate::config::AgentKind::Command,
12171 model: None,
12172 command: vec!["true".to_owned()],
12173 extra_args: Vec::new(),
12174 env: BTreeMap::new(),
12175 prompt_delivery: None,
12176 max_turn_secs: None,
12177 }
12178 }
12179
12180 fn ids(xs: &[&str]) -> BTreeSet<String> {
12181 xs.iter().map(|s| (*s).to_owned()).collect()
12182 }
12183
12184 #[test]
12185 fn pick_successor_skips_an_agent_another_seat_holds() {
12186 let roster = [spec("a"), spec("b"), spec("c")];
12188 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12189 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12190 }
12191
12192 #[test]
12193 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12194 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12196 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12197 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12198 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12199 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12200 }
12201
12202 #[test]
12203 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12204 let roster = [spec("a"), spec("b")];
12205 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12206 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12207 let carried = ids(&["b"]);
12208 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12209 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12210 }
12211
12212 #[test]
12213 fn pick_successor_returns_none_without_any_untried_successor() {
12214 let roster = [spec("a"), spec("b")];
12215 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12216 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12217 let carried = ids(&["a"]);
12218 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12219 }
12220
12221 #[test]
12222 fn pick_successor_rescue_avoids_another_seats_occupant() {
12223 let roster = [spec("a"), spec("b"), spec("c")];
12225 let carried = ids(&["b", "c"]);
12226 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12227 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12228 }
12229
12230 #[test]
12231 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12232 let roster = [spec("a"), spec("b"), spec("c")];
12233 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12234 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12235 }
12236
12237 #[test]
12238 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12239 let roster = [spec("a"), spec("b"), spec("c")];
12240 let failed = ids(&["a", "b", "c"]);
12241 let mut tried = ids(&["b"]);
12243 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12244 assert_eq!(first.id, "a");
12245 tried.insert(first.id.clone());
12246 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12247 assert_eq!(second.id, "c");
12248 tried.insert(second.id.clone());
12249 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12250 }
12251
12252 #[test]
12253 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12254 let roster = [spec("a"), spec("b")];
12255 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12256 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12257 }
12258
12259 #[test]
12260 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12261 let roster = [spec("a"), spec("b"), spec("c")];
12262 let h = SeatHistory {
12263 failed: ids(&["a"]),
12264 last_ok: Some("b".to_owned()),
12265 last_fail: None,
12266 };
12267 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12268 let gone = SeatHistory {
12270 last_ok: Some("zzz".to_owned()),
12271 ..h.clone()
12272 };
12273 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12274 let failed = SeatHistory {
12275 failed: ids(&["a", "b"]),
12276 last_ok: Some("b".to_owned()),
12277 last_fail: None,
12278 };
12279 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12280 }
12281
12282 #[test]
12283 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12284 let first = SeatState::new("review-1", "alpha", 7);
12285 let next = handover_seat("review-1", "gamma", 7);
12286 assert_ne!(first.claude_session, next.claude_session);
12287 }
12288
12289 #[test]
12290 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12291 let mut state = state_with_summary("x", "y");
12292 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12293 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12294 assert_ne!(a.claude_session, b.claude_session);
12295 }
12296
12297 #[test]
12298 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12299 let roster = [spec("a"), spec("b")];
12300 let h = SeatHistory {
12301 failed: ids(&["a", "b"]),
12302 ..SeatHistory::default()
12303 };
12304 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12305 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12306 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12307 }
12308
12309 #[test]
12316 fn failure_signature_ignores_numbers_and_paths() {
12317 assert_eq!(
12318 failure_signature("exited with Some(2) and no usable output"),
12319 failure_signature("exited with Some(137) and no usable output")
12320 );
12321 assert_eq!(
12322 failure_signature("process exited with no usable output (exit code 2)"),
12323 failure_signature("process exited with no usable output (exit code 137)")
12324 );
12325 for (old, new) in [
12326 (
12327 "exited with Some(2) and no usable output",
12328 "process exited with no usable output (exit code 2)",
12329 ),
12330 (
12331 "exited with None and no usable output",
12332 "process exited with no usable output (exit code unavailable)",
12333 ),
12334 ] {
12335 let previous = FailClass::Other(failure_signature(old));
12336 let current = FailClass::Other(failure_signature(new));
12337 assert_eq!(previous, current, "resumption retains the failure class");
12338 assert!(!should_hand_over(Some(&previous), ¤t));
12339 }
12340 assert_eq!(
12341 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12342 failure_signature("cannot open /var/x.txt: denied")
12343 );
12344 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12345 }
12346
12347 #[test]
12348 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12349 let other = FailClass::Other("x".into());
12350 assert!(should_hand_over(None, &FailClass::Quota));
12351 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12352 assert!(should_hand_over(
12353 Some(&FailClass::Timeout),
12354 &FailClass::Timeout
12355 ));
12356 assert!(should_hand_over(None, &other));
12357 assert!(!should_hand_over(Some(&other), &other));
12358 assert!(should_hand_over(
12359 Some(&other),
12360 &FailClass::Other("y".into())
12361 ));
12362 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12364 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12365 }
12366
12367 #[test]
12368 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12369 let a = SeatState::new("judge-1", "alpha", 7);
12370 let b = handover_seat("judge-1", "beta", 7);
12371 assert_ne!(a.claude_session, b.claude_session);
12372 assert_eq!(b.turns, 0);
12373 }
12374
12375 #[test]
12376 fn a_timeout_is_classified_apart_from_other_failures() {
12377 assert_eq!(
12378 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12379 Some(FailClass::Timeout)
12380 );
12381 assert!(matches!(
12382 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12383 Some(FailClass::Other(_))
12384 ));
12385 }
12386
12387 #[test]
12388 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12389 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12390 let tried = BTreeSet::from(["beta".to_owned()]);
12391 let next = next_untried_in_roster(&roster, 1, &tried);
12394 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12395 }
12396
12397 #[test]
12398 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12399 let roster = vec![spec("alpha"), spec("beta")];
12400 let tried = BTreeSet::from(["beta".to_owned()]);
12401 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12405 }
12406
12407 #[test]
12408 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12409 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12410 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12411 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12415 }
12416
12417 #[test]
12418 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12419 let roster = vec![spec("a"), spec("a"), spec("b")];
12420 let tried = BTreeSet::from(["a".to_owned()]);
12421 let next = next_untried_in_roster(&roster, 0, &tried);
12422 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12423 }
12424
12425 #[test]
12426 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12427 let roster = vec![spec("a"), spec("b")];
12428 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12429 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12430 }
12431
12432 #[test]
12433 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12434 let dir = tempfile::tempdir().unwrap();
12435 let bay = dir.path().join("ffff");
12436
12437 remove_if_empty(&bay);
12439 assert!(!bay.exists());
12440
12441 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12444 remove_if_empty(&bay);
12445 assert!(bay.exists(), "non-empty directory must survive");
12446
12447 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12449 remove_if_empty(&bay);
12450 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12451 }
12452
12453 #[test]
12462 fn a_full_panel_that_found_nothing_is_clean() {
12463 assert!(round_is_clean(
12464 0,
12465 true,
12466 2,
12467 2,
12468 0,
12469 IncompleteReviewPolicy::Block
12470 ));
12471 }
12472
12473 #[test]
12474 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12475 assert!(!round_is_clean(
12476 0,
12477 true,
12478 1,
12479 2,
12480 0,
12481 IncompleteReviewPolicy::Block
12482 ));
12483 }
12484
12485 #[test]
12486 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12487 assert!(!round_is_clean(
12488 1,
12489 true,
12490 1,
12491 2,
12492 0,
12493 IncompleteReviewPolicy::Warn
12494 ));
12495 }
12496
12497 #[test]
12498 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12499 assert!(round_is_clean(
12500 0,
12501 true,
12502 1,
12503 2,
12504 0,
12505 IncompleteReviewPolicy::Warn
12506 ));
12507 }
12508
12509 #[test]
12510 fn a_full_panel_with_an_open_finding_is_not_clean() {
12511 assert!(!round_is_clean(
12512 1,
12513 true,
12514 2,
12515 2,
12516 0,
12517 IncompleteReviewPolicy::Block
12518 ));
12519 }
12520
12521 #[test]
12522 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12523 assert!(!round_is_clean(
12524 0,
12525 false,
12526 2,
12527 2,
12528 0,
12529 IncompleteReviewPolicy::Block
12530 ));
12531 }
12532
12533 #[test]
12540 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12541 assert!(round_is_clean(
12544 0,
12545 true,
12546 1,
12547 2,
12548 1,
12549 IncompleteReviewPolicy::Block
12550 ));
12551 }
12552
12553 #[test]
12554 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12555 assert!(!round_is_clean(
12558 0,
12559 true,
12560 1,
12561 2,
12562 0,
12563 IncompleteReviewPolicy::Block
12564 ));
12565 }
12566
12567 #[test]
12568 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12569 assert!(!round_is_clean(
12570 1,
12571 true,
12572 1,
12573 2,
12574 1,
12575 IncompleteReviewPolicy::Block
12576 ));
12577 assert!(!round_is_clean(
12578 0,
12579 false,
12580 1,
12581 2,
12582 1,
12583 IncompleteReviewPolicy::Block
12584 ));
12585 }
12586
12587 #[test]
12588 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12589 assert!(!round_is_clean(
12593 0,
12594 true,
12595 0,
12596 2,
12597 2,
12598 IncompleteReviewPolicy::Block
12599 ));
12600 }
12601
12602 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12603 CommandOutcome {
12604 command: "test".to_owned(),
12605 code,
12606 output_tail: String::new(),
12607 duration_ms: 0,
12608 resource_blocked,
12609 }
12610 }
12611
12612 #[test]
12613 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12614 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12615 assert!(
12616 !verify_inconclusive(&[outcome(Some(1), false)]),
12617 "an ordinary failure is still evidence about the patch"
12618 );
12619 assert!(verify_inconclusive(&[outcome(None, true)]));
12620 assert!(
12621 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12622 "one inconclusive outcome taints the whole batch"
12623 );
12624 assert!(!verify_inconclusive(&[]));
12625 }
12626
12627 #[tokio::test]
12628 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12629 let calls = std::sync::atomic::AtomicUsize::new(0);
12633 let started = Instant::now();
12634 wait_for_pids_with(
12635 &[123],
12636 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12637 Duration::from_millis(5),
12638 Duration::from_secs(5),
12639 )
12640 .await;
12641 assert!(
12642 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12643 "must keep checking rather than deciding on the first answer"
12644 );
12645 assert!(
12646 started.elapsed() < Duration::from_secs(1),
12647 "must return the moment it is confirmed dead, not wait out the ceiling"
12648 );
12649 }
12650
12651 #[tokio::test]
12652 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12653 let started = Instant::now();
12654 wait_for_pids_with(
12655 &[123],
12656 |_| true, Duration::from_millis(5),
12658 Duration::from_millis(30),
12659 )
12660 .await;
12661 let elapsed = started.elapsed();
12662 assert!(
12663 elapsed >= Duration::from_millis(30),
12664 "must not give up before its own ceiling: {elapsed:?}"
12665 );
12666 assert!(
12667 elapsed < Duration::from_secs(1),
12668 "must not wait past its own ceiling either: {elapsed:?}"
12669 );
12670 }
12671
12672 #[tokio::test]
12673 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12674 let started = Instant::now();
12675 wait_for_pids_with(
12676 &[],
12677 |_| true,
12678 Duration::from_secs(5),
12679 Duration::from_secs(5),
12680 )
12681 .await;
12682 assert!(
12683 started.elapsed() < Duration::from_millis(200),
12684 "an empty pid list has nothing to confirm"
12685 );
12686 }
12687
12688 fn review_round(
12694 clean: bool,
12695 blocking: usize,
12696 answered: usize,
12697 expected: usize,
12698 progressed: bool,
12699 e2e_ok: bool,
12700 ) -> ReviewRound {
12701 ReviewRound {
12702 round: 1,
12703 head: "h".to_owned(),
12704 verified_head: None,
12705 verified_at: None,
12706 reviews: Vec::new(),
12707 e2e: vec![CommandOutcome {
12708 command: "test".to_owned(),
12709 code: Some(if e2e_ok { 0 } else { 1 }),
12710 output_tail: String::new(),
12711 duration_ms: 0,
12712 resource_blocked: false,
12713 }],
12714 verify_retried: false,
12715 e2e_deferred: false,
12716 e2e_defer_reason: None,
12717 fix: None,
12718 blocking,
12719 answered,
12720 expected,
12721 clean,
12722 progressed,
12723 vote_split: false,
12724 reconsideration: Vec::new(),
12725 verdict: None,
12726 }
12727 }
12728
12729 #[test]
12730 fn review_conclusion_is_none_when_nothing_has_run() {
12731 assert_eq!(review_conclusion(&[], 3), None);
12732 }
12733
12734 #[test]
12735 fn review_conclusion_is_none_while_rounds_remain() {
12736 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12737 assert_eq!(review_conclusion(&rounds, 3), None);
12738 }
12739
12740 #[test]
12741 fn review_conclusion_is_gating_once_a_round_is_clean() {
12742 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12743 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12744 }
12745
12746 #[test]
12747 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12748 let rounds = vec![
12749 review_round(false, 1, 2, 2, true, true),
12750 review_round(false, 1, 2, 2, true, true),
12751 ];
12752 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12753 }
12754
12755 #[test]
12756 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12757 let rounds = vec![
12758 review_round(false, 1, 2, 2, true, true),
12759 review_round(false, 1, 2, 2, true, false),
12760 ];
12761 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12762 }
12763
12764 #[test]
12765 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12766 let mut blocked = review_round(false, 1, 2, 2, true, false);
12773 blocked.e2e[0].resource_blocked = true;
12774 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12775 assert_eq!(review_conclusion(&rounds, 2), None);
12776 }
12777
12778 #[test]
12779 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12780 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12782 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12783 }
12784
12785 #[test]
12786 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12787 let rounds = vec![
12788 review_round(false, 1, 2, 2, false, true),
12789 review_round(false, 1, 2, 2, false, true),
12790 ];
12791 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12792 }
12793
12794 fn secs(n: u64) -> Duration {
12795 Duration::from_secs(n)
12796 }
12797
12798 #[tokio::test]
12801 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12802 ask_test_home();
12803 let tmp = tempfile::tempdir().unwrap();
12804 let repo = tmp.path().join("repo");
12805 std::fs::create_dir_all(&repo).unwrap();
12806 init_repo(&repo);
12807 let g = |args: &[&str]| {
12808 let out = std::process::Command::new("git")
12809 .args(args)
12810 .current_dir(&repo)
12811 .quiet()
12812 .output()
12813 .unwrap();
12814 assert!(
12815 out.status.success(),
12816 "{}",
12817 String::from_utf8_lossy(&out.stderr)
12818 );
12819 };
12820 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12821 g(&["checkout", "-b", "historical"]);
12822 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12823 g(&["add", "."]);
12824 g(&["commit", "-m", "old alternative"]);
12825 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12826 g(&["checkout", "-b", "magi/task-selected", &base]);
12827 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12828 g(&["add", "."]);
12829 g(&["commit", "-m", "measurement infrastructure"]);
12830 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12831 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12832 g(&["add", "."]);
12833 g(&["commit", "-m", "dependent implementation"]);
12834 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12835 g(&["checkout", "main"]);
12836 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12837 g(&["add", "."]);
12838 g(&["commit", "-m", "unrelated base change"]);
12839 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12840 let origin = tmp.path().join("origin.git");
12841 g(&[
12842 "clone",
12843 "--bare",
12844 repo.to_str().unwrap(),
12845 origin.to_str().unwrap(),
12846 ]);
12847 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12848 g(&["fetch", "origin"]);
12849 let selection = crate::pr_recovery::Selection {
12850 url: "https://example.test/project/pull/1".into(),
12851 branch: "magi/task-selected".into(),
12852 head: head.clone(),
12853 source_head: None,
12854 source_run: None,
12855 source_proof: None,
12856 disposition: Some(crate::pr_recovery::Disposition::Review),
12857 instruction: "preserve measured behavior".into(),
12858 approved_at: jiff::Timestamp::now(),
12859 };
12860 let mut template = runner_at(RunStatus::Reviewing);
12861 template.state.repo = repo.clone();
12862 template.state.base_commit = current.clone();
12863 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12864 template.state.origin = Some(crate::run::Origin::queue("selected"));
12865 template.state.pr_head = Some(selection.branch.clone());
12866 template.state.pr_recovery = Some(selection.clone());
12867 let mut runner =
12868 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12869 .await
12870 .unwrap();
12871 let wt = runner.state.winner().unwrap().worktree.clone();
12872 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12873 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12874 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12875 assert!(runner.state.seeds.is_empty());
12876 assert_eq!(runner.state.instruction, "preserve measured behavior");
12877 runner.sync_to_base().await.unwrap();
12878 assert_eq!(
12879 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12880 "prerequisite"
12881 );
12882 assert_eq!(
12883 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12884 "measured implementation"
12885 );
12886 assert_eq!(
12887 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12888 "current main change"
12889 );
12890 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12891 assert_eq!(
12892 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12893 head
12894 );
12895
12896 let mut partial = runner_at(RunStatus::Reviewing);
12898 partial.state.repo = repo.clone();
12899 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12900 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12901 run: Some("exit 1".into()),
12902 ..Default::default()
12903 }];
12904 let selection = runner.state.pr_recovery.clone().unwrap();
12905 partial.state.pr_recovery = Some(selection.clone());
12906 partial.state.pr_head = Some(selection.branch.clone());
12907 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12908 let id = partial.state.id.clone();
12909 let error = match Runner::open_selected(
12910 &repo,
12911 partial.state,
12912 partial.roles,
12913 1,
12914 runner.state.base_commit.clone(),
12915 selection,
12916 )
12917 .await
12918 {
12919 Ok(_) => panic!("setup must fail"),
12920 Err(e) => e,
12921 };
12922 assert_eq!(
12923 error
12924 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12925 .unwrap()
12926 .run,
12927 id
12928 );
12929 let failed = RunState::load(&id).unwrap();
12930 assert_eq!(failed.status, RunStatus::Blocked);
12931 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12932 assert!(Runner::resume(&id).is_err());
12933 runner.state.status = RunStatus::Ready;
12934 runner.state.driver_exited = true;
12935 runner.state.save().unwrap();
12936 let mut retry = runner_at(RunStatus::Reviewing);
12937 retry.state.repo = repo.clone();
12938 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12939 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12940 reselected.approved_at = jiff::Timestamp::now();
12941 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12942 retry.state.pr_head = Some(reselected.branch.clone());
12943 retry.state.pr_recovery = Some(reselected.clone());
12944 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12945 .await
12946 .unwrap();
12947 let restarted = Runner::open_selected(
12948 &repo,
12949 retry.state,
12950 retry.roles,
12951 1,
12952 runner.state.base_commit.clone(),
12953 reselected,
12954 )
12955 .await
12956 .unwrap();
12957 assert_eq!(
12958 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12959 .await
12960 .unwrap(),
12961 head
12962 );
12963 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12964 assert_eq!(
12965 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12966 head
12967 );
12968 }
12969
12970 #[tokio::test]
12971 #[cfg(unix)]
12972 async fn fallback_and_interrupted_restoration_require_coordination() {
12973 use std::os::fd::AsRawFd as _;
12974 ask_test_home();
12975 let tmp = tempfile::tempdir().unwrap();
12976 let mut runner = registered_runner(tmp.path()).await;
12977 runner.state.config.graph.timeout_resource_wait = 1;
12978 let winner = runner.state.candidates[0].clone();
12979 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12980 let leftover = winner.worktree.join("leftover");
12981 std::fs::write(&leftover, "untrusted edits").unwrap();
12982 let next = tmp.path().join("next-fixer");
12983 let mut failed = spec("first");
12984 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12985 let mut successor = spec("second");
12986 successor.command = vec![
12987 "sh".into(),
12988 "-c".into(),
12989 format!("touch '{}'; echo done", next.display()),
12990 ];
12991 let attempts = vec![
12992 (failed.clone(), "first".into()),
12993 (successor.clone(), "second".into()),
12994 ];
12995 let lock_path = runner.state.repo.join(".git/updater.lock");
12996 let external = std::fs::File::create(&lock_path).unwrap();
12997 assert_eq!(
12998 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12999 0
13000 );
13001 let artifacts = tmp.path().join("artifacts");
13002 let build = |spec, seat| SeatJob {
13003 spec,
13004 seat,
13005 cwd: winner.worktree.clone(),
13006 prompt: "fixture".into(),
13007 timeout: Duration::from_secs(10),
13008 allow_write: true,
13009 sessions: false,
13010 artifacts: artifacts.clone(),
13011 stem: "fix".into(),
13012 handover: None,
13013 };
13014 let (_, _, out) = runner
13015 .ask_fixer_among(
13016 attempts.clone(),
13017 "graft",
13018 None,
13019 Some((&winner.worktree, &head)),
13020 build,
13021 )
13022 .await;
13023 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13024 assert!(leftover.exists(), "contention must prevent clean");
13025 assert!(!next.exists());
13026 assert!(runner.state.handovers.is_empty());
13027 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13029 from_run: "earlier".into(),
13030 generation: 1,
13031 mode: crate::recovery::Mode::VerifyRepair,
13032 reasons: Vec::new(),
13033 reviewed_head: None,
13034 rounds: Vec::new(),
13035 open_findings: Vec::new(),
13036 evidence: None,
13037 criteria_inherited: true,
13038 repair: crate::recovery::Repair::Started {
13039 before: head.clone(),
13040 },
13041 });
13042 assert!(
13043 runner
13044 .recovery_repair(&winner)
13045 .await
13046 .unwrap_err()
13047 .is::<crate::repo_lock::ResourceBlocked>()
13048 );
13049 assert!(matches!(
13050 runner.state.recovery.as_ref().unwrap().repair,
13051 crate::recovery::Repair::Started { .. }
13052 ));
13053 assert!(leftover.exists());
13054 drop(external);
13055 runner.recovery_repair(&winner).await.unwrap();
13056 assert!(!leftover.exists());
13057 assert!(matches!(
13058 runner.state.recovery.as_ref().unwrap().repair,
13059 crate::recovery::Repair::Done { .. }
13060 ));
13061 std::fs::remove_file(&lock_path).unwrap();
13063 std::fs::create_dir(&lock_path).unwrap();
13064 std::fs::write(&leftover, "untrusted edits").unwrap();
13065 let (_, _, out) = runner
13066 .ask_fixer_among(
13067 attempts,
13068 "graft",
13069 None,
13070 Some((&winner.worktree, &head)),
13071 build,
13072 )
13073 .await;
13074 assert!(matches!(out, AgentOutcome::Failed(_)));
13075 assert!(leftover.exists());
13076 assert!(!next.exists());
13077 assert_eq!(
13078 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13079 head
13080 );
13081 }
13082
13083 #[cfg(unix)]
13084 async fn registered_runner(root: &Path) -> Runner {
13085 ask_test_home();
13086 let repo = root.join("repo");
13087 std::fs::create_dir_all(&repo).unwrap();
13088 init_repo(&repo);
13089 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13090 let mut runner = runner_at(RunStatus::Prep);
13091 runner.state.repo = repo.clone();
13092 runner.state.config.coordination = crate::config::test_coordination();
13093 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13094 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13095 runner.state.config.graph.advise = false;
13096 runner.roles.implementers = vec![spec("implement")];
13097 runner.roles.judges = vec![spec("judge")];
13098 runner.prep().await.unwrap();
13099 runner
13100 }
13101
13102 #[tokio::test]
13103 #[cfg(unix)]
13104 async fn parked_implementer_persists_completed_work() {
13105 for registered in [false, true] {
13106 let tmp = tempfile::tempdir().unwrap();
13107 let mut runner = registered_runner(tmp.path()).await;
13108 if !registered {
13109 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13110 }
13111 let marker = tmp.path().join("started");
13112 let release = tmp.path().join("release");
13113 let mut implementer = spec("implement");
13114 implementer.command = vec![
13115 "sh".into(),
13116 "-c".into(),
13117 format!(
13118 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13119 marker.display(),
13120 release.display()
13121 ),
13122 ];
13123 runner.roles.implementers = vec![implementer.clone()];
13124 runner.state.config.agents = vec![implementer];
13125 let pause = runner.pause.clone();
13126 let (result, ()) = tokio::join!(runner.implement(), async {
13127 tokio::time::timeout(Duration::from_secs(10), async {
13128 while !marker.exists() {
13129 tokio::time::sleep(Duration::from_millis(10)).await;
13130 }
13131 })
13132 .await
13133 .unwrap();
13134 pause.park();
13135 std::fs::write(&release, "go").unwrap();
13136 });
13137 result.unwrap();
13138 let candidate = &runner.state.candidates[0];
13139 assert!(candidate.commits > 0, "{candidate:?}");
13140 assert!(candidate.summary.contains("saved-result"));
13141 assert_eq!(candidate.files, 1);
13142 let before = candidate.commits;
13143 runner.implement().await.unwrap();
13144 assert_eq!(runner.state.candidates[0].commits, before);
13145 }
13146 }
13147
13148 #[tokio::test]
13149 #[cfg(unix)]
13150 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13151 let tmp = tempfile::tempdir().unwrap();
13152 let mut runner = registered_runner(tmp.path()).await;
13153 let winner = runner.state.candidates[0].clone();
13154 let cache = tmp.path().join("cache");
13155 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13156 runner.state.config.graph.timeout_resource_wait = 0;
13157 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13158 let crate::cache::AcquireOutcome::Acquired(held) =
13159 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13160 else {
13161 panic!("cache must be free")
13162 };
13163 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13164 from_run: "earlier".into(),
13165 generation: 1,
13166 mode: crate::recovery::Mode::VerifyRepair,
13167 reasons: Vec::new(),
13168 reviewed_head: None,
13169 rounds: Vec::new(),
13170 open_findings: Vec::new(),
13171 evidence: None,
13172 criteria_inherited: true,
13173 repair: crate::recovery::Repair::NotStarted,
13174 });
13175 runner.recovery_repair(&winner).await.unwrap();
13176 assert!(matches!(
13177 runner.state.recovery.as_ref().unwrap().repair,
13178 crate::recovery::Repair::WaitingForRestore { .. }
13179 ));
13180 drop(held);
13181 runner.state.resource_block = None;
13182 runner.state.config.graph.timeout_resource_wait = 1;
13183 runner.recovery_repair(&winner).await.unwrap();
13184 assert!(matches!(
13185 runner.state.recovery.as_ref().unwrap().repair,
13186 crate::recovery::Repair::Done { .. }
13187 ));
13188 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13189 }
13190
13191 #[tokio::test]
13192 #[cfg(unix)]
13193 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13194 use std::os::fd::AsRawFd as _;
13195 for graft in [false, true] {
13196 let tmp = tempfile::tempdir().unwrap();
13197 let mut runner = registered_runner(tmp.path()).await;
13198 runner.state.config.graph.timeout_resource_wait = 1;
13199 let winner = runner.state.candidates[0].clone();
13200 let marker = tmp.path().join("fixer-started");
13201 let mut fixer = spec("fixer");
13202 fixer.command = vec![
13203 "sh".into(),
13204 "-c".into(),
13205 format!("echo started >> '{}'; echo done", marker.display()),
13206 ];
13207 runner.roles.fixer = Some(vec![fixer]);
13208 if graft {
13209 let mut loser = winner.clone();
13210 loser.label = 'B';
13211 loser.index = 1;
13212 runner.state.candidates.push(loser);
13213 runner.state.config.graph.graft_runner_up = true;
13214 runner.state.tally = Some(Tally {
13215 first_choice: BTreeMap::new(),
13216 borda: BTreeMap::new(),
13217 winner: winner.label,
13218 rankings: 1,
13219 unanimous_initial: true,
13220 deliberated: false,
13221 changed_votes: 0,
13222 unanimous_final: true,
13223 tie_break: None,
13224 judges: 1,
13225 present: 1,
13226 quorum: 1,
13227 met_quorum: true,
13228 uncontested: None,
13229 });
13230 runner.state.judgements.push(crate::run::Judgement {
13231 judge: 1,
13232 seat: "judge-1".into(),
13233 agent: "judge".into(),
13234 ranking: vec![winner.label, 'B'],
13235 reasons: BTreeMap::new(),
13236 confidence: None,
13237 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13238 order: vec![0, 1],
13239 failed: None,
13240 duration_ms: 0,
13241 });
13242 } else {
13243 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13244 from_run: "earlier".into(),
13245 generation: 1,
13246 mode: crate::recovery::Mode::VerifyRepair,
13247 reasons: Vec::new(),
13248 reviewed_head: None,
13249 rounds: Vec::new(),
13250 open_findings: Vec::new(),
13251 evidence: None,
13252 criteria_inherited: true,
13253 repair: crate::recovery::Repair::NotStarted,
13254 });
13255 }
13256 let reader = crate::repo_lock::SeatLease::for_checkout(
13257 &winner.worktree,
13258 false,
13259 crate::config::test_coordination().as_ref(),
13260 )
13261 .await
13262 .unwrap();
13263 let pause = runner.pause.clone();
13264 let (result, ()) = tokio::join!(
13265 async {
13266 if graft {
13267 runner.graft().await
13268 } else {
13269 runner.recovery_repair(&winner).await
13270 }
13271 },
13272 async {
13273 tokio::time::sleep(Duration::from_millis(100)).await;
13274 pause.park();
13275 }
13276 );
13277 result.unwrap();
13278 assert!(!marker.exists(), "a blocked seat must not spawn");
13279 runner.state = RunState::load(&runner.state.id).unwrap();
13281 if graft {
13282 assert_eq!(
13283 runner.state.graft.as_ref().unwrap().outcome,
13284 GraftOutcome::WaitingForRestore
13285 );
13286 } else {
13287 assert!(matches!(
13288 runner.state.recovery.as_ref().unwrap().repair,
13289 crate::recovery::Repair::WaitingForRestore { .. }
13290 ));
13291 }
13292 drop(reader);
13293 runner.pause = Pause::new();
13294 runner.state.resource_block = None;
13295 let external =
13297 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13298 assert_eq!(
13299 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13300 0
13301 );
13302 let leftover = winner.worktree.join("leftover");
13303 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13304 let result = if graft {
13305 runner.graft().await
13306 } else {
13307 runner.recovery_repair(&winner).await
13308 };
13309 assert!(
13310 result
13311 .unwrap_err()
13312 .is::<crate::repo_lock::ResourceBlocked>()
13313 );
13314 assert!(leftover.exists());
13315 assert!(!marker.exists());
13316 drop(external);
13317 runner.state.resource_block = None;
13318 let result = if graft {
13319 runner.graft().await
13320 } else {
13321 runner.recovery_repair(&winner).await
13322 };
13323 result.unwrap();
13324 assert!(!leftover.exists(), "restore must precede retry");
13325 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13326 if graft {
13327 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13328 } else {
13329 assert!(matches!(
13330 runner.state.recovery.as_ref().unwrap().repair,
13331 crate::recovery::Repair::Done { .. }
13332 ));
13333 }
13334 }
13335 }
13336
13337 #[tokio::test]
13338 #[cfg(unix)]
13339 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13340 use std::os::fd::AsRawFd as _;
13341 let tmp = tempfile::tempdir().unwrap();
13342 let mut runner = registered_runner(tmp.path()).await;
13343 runner.state.config.graph.timeout_resource_wait = 1;
13344 let winner = runner.state.candidates[0].clone();
13345 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13346 runner.state.tally = Some(Tally {
13347 first_choice: BTreeMap::new(),
13348 borda: BTreeMap::new(),
13349 winner: winner.label,
13350 rankings: 0,
13351 unanimous_initial: false,
13352 deliberated: false,
13353 changed_votes: 0,
13354 unanimous_final: false,
13355 tie_break: None,
13356 judges: 0,
13357 present: 0,
13358 quorum: 0,
13359 met_quorum: true,
13360 uncontested: Some("fixture".into()),
13361 });
13362 runner.state.graft = Some(GraftRecord {
13363 pre: pre.clone(),
13364 post: None,
13365 lines: None,
13366 limit: None,
13367 agent: None,
13368 outcome: GraftOutcome::Running,
13369 requested: Vec::new(),
13370 adoption: None,
13371 adoption_error: None,
13372 });
13373 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13374 git::commit_all(&winner.worktree, "unfinished graft")
13375 .await
13376 .unwrap();
13377 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13378 let leftover = winner.worktree.join("leftover");
13379 std::fs::write(&leftover, "untrusted edits").unwrap();
13380 let path = runner.state.repo.join(".git/updater.lock");
13381 let external = std::fs::File::create(&path).unwrap();
13382 assert_eq!(
13383 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13384 0
13385 );
13386 assert!(
13387 runner
13388 .graft()
13389 .await
13390 .unwrap_err()
13391 .is::<crate::repo_lock::ResourceBlocked>()
13392 );
13393 assert!(matches!(
13394 runner.state.graft.as_ref().unwrap().outcome,
13395 GraftOutcome::Running
13396 ));
13397 assert_eq!(
13398 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13399 unfinished
13400 );
13401 assert!(leftover.exists());
13402 drop(external);
13403 std::fs::remove_file(&path).unwrap();
13404 std::fs::create_dir(&path).unwrap();
13405 assert!(runner.graft().await.is_err());
13406 assert!(matches!(
13407 runner.state.graft.as_ref().unwrap().outcome,
13408 GraftOutcome::Running
13409 ));
13410 assert_eq!(
13411 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13412 unfinished
13413 );
13414 assert!(leftover.exists());
13415 std::fs::remove_dir(&path).unwrap();
13416 runner.graft().await.unwrap();
13417 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13418 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13419 assert!(!leftover.exists());
13420 }
13421
13422 #[tokio::test]
13423 #[cfg(unix)]
13424 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13425 use std::os::fd::AsRawFd as _;
13426 let tmp = tempfile::tempdir().unwrap();
13427 let mut runner = registered_runner(tmp.path()).await;
13428 runner.state.prep_complete = Some(false);
13429 runner.state.config.graph.timeout_resource_wait = 1;
13430 runner.state.save().unwrap();
13431 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13432 assert_eq!(
13433 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13434 0
13435 );
13436 runner.execute().await.unwrap();
13437 let disk = RunState::load(&runner.state.id).unwrap();
13438 assert!(disk.driver_exited);
13439 assert_eq!(disk.driver_pid, None);
13440 assert_eq!(runner.state.status, RunStatus::Blocked);
13441 assert_eq!(
13442 runner.state.resource_block.as_ref().unwrap().kind,
13443 "repository coordination"
13444 );
13445 assert!(runner.state.prep_failure.is_none());
13446 assert!(runner.state.seats.is_empty());
13447 runner.pause.park();
13448 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13451 .await
13452 .unwrap()
13453 .unwrap();
13454 assert!(runner.state.parked);
13455 assert!(runner.state.prep_failure.is_none());
13456 let disk = RunState::load(&runner.state.id).unwrap();
13457 assert!(disk.driver_exited);
13458 assert_eq!(disk.driver_pid, None);
13459 }
13460
13461 #[tokio::test]
13462 #[cfg(unix)]
13463 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13464 use std::os::fd::AsRawFd as _;
13465 let tmp = tempfile::tempdir().unwrap();
13466 let mut runner = registered_runner(tmp.path()).await;
13467 runner.state.prep_complete = Some(false);
13468 runner.state.config.graph.timeout_resource_wait = 30;
13469 runner.state.save().unwrap();
13470 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13471 assert_eq!(
13472 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13473 0
13474 );
13475 assert!(
13476 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13477 .await
13478 .is_err()
13479 );
13480 let disk = RunState::load(&runner.state.id).unwrap();
13481 assert!(disk.driver_exited);
13482 assert_eq!(disk.driver_pid, None);
13483 assert!(disk.status.resumable());
13484 assert!(runner.state.driver_exited);
13485 assert_eq!(runner.state.driver_pid, None);
13486 runner.state.save().unwrap();
13487 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13488 }
13489
13490 #[tokio::test]
13491 #[cfg(unix)]
13492 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13493 let tmp = tempfile::tempdir().unwrap();
13494 let mut runner = registered_runner(tmp.path()).await;
13495 let winner = runner.state.candidates[0].clone();
13496 let cache = tmp.path().join("cache");
13497 let started = tmp.path().join("pre-gate-started");
13498 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13499 runner.state.config.verify.gate =
13500 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13501 let home = crate::run::home();
13502 let owner =
13503 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13504 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13505 crate::cache::AcquireOutcome::Acquired(g) => g,
13506 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13507 };
13508 let mut gate = Box::pin(runner.run_gate(&winner));
13509 assert!(
13510 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13511 .await
13512 .is_err()
13513 );
13514 assert!(!started.exists(), "pre-gate must wait for the cache too");
13515 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13519 let editor = crate::repo_lock::SeatLease::checkout_with(
13520 &winner.worktree,
13521 true,
13522 crate::config::test_coordination().as_ref(),
13523 &wait,
13524 )
13525 .await
13526 .unwrap();
13527 drop(editor);
13528 guard.release();
13529 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13530 .await
13531 .unwrap()
13532 .unwrap();
13533 drop(gate);
13534 assert!(outcomes.iter().all(CommandOutcome::ok));
13535 assert!(started.exists());
13536 }
13537
13538 #[tokio::test]
13539 #[cfg(unix)]
13540 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13541 use std::os::fd::AsRawFd as _;
13542
13543 ask_test_home();
13544 let tmp = tempfile::tempdir().unwrap();
13545 let repo = tmp.path().join("repo");
13546 std::fs::create_dir_all(&repo).unwrap();
13547 init_repo(&repo);
13548 let common = repo.join(".git");
13549 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13550 let mut runner = runner_at(RunStatus::Prep);
13551 runner.state.repo = repo.clone();
13552 runner.state.config.coordination = crate::config::test_coordination();
13553 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13554 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13555 runner.state.config.graph.advise = false;
13556 runner.roles.implementers = vec![spec("implement")];
13557 runner.roles.judges = vec![spec("judge")];
13558 runner.prep().await.unwrap();
13559 let winner = runner.state.candidates[0].clone();
13560 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13563 let completed = tmp.path().join("seat-completed");
13564 runner.roles.implementers[0].command = vec![
13565 "sh".into(),
13566 "-c".into(),
13567 format!("touch '{}'; echo completed", completed.display()),
13568 ];
13569 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13570 assert_eq!(
13571 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13572 0
13573 );
13574 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13575 let mut implement = Box::pin(runner.implement());
13576 tokio::select! {
13577 result = &mut implement => panic!("rescue must wait: {result:?}"),
13578 result = tokio::time::timeout(Duration::from_secs(10), async {
13579 while !completed.exists() {
13580 tokio::time::sleep(Duration::from_millis(20)).await;
13581 }
13582 }) => result.unwrap(),
13583 }
13584 assert!(
13585 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13586 .await
13587 .is_err()
13588 );
13589 assert_eq!(
13590 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13591 before
13592 );
13593 assert_eq!(
13594 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13595 0
13596 );
13597 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13598 .await
13599 .unwrap()
13600 .unwrap();
13601 drop(implement);
13602 assert_ne!(
13603 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13604 before
13605 );
13606 assert!(git::is_clean(&winner.worktree).await.unwrap());
13607
13608 let started = tmp.path().join("pre-gate-started");
13609 let proceed = tmp.path().join("pre-gate-proceed");
13610 let checked = tmp.path().join("gate-ran");
13611 runner.state.config.verify.pre_gate = vec![format!(
13612 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13613 started.display(),
13614 proceed.display()
13615 )];
13616 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13617 assert_eq!(
13618 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13619 0
13620 );
13621 let mut gate = Box::pin(runner.run_gate(&winner));
13622 assert!(
13623 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13624 .await
13625 .is_err()
13626 );
13627 assert!(!started.exists());
13628 assert!(!checked.exists());
13629 assert!(git::is_clean(&winner.worktree).await.unwrap());
13630 assert_eq!(
13631 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13632 0
13633 );
13634 tokio::select! {
13635 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13636 result = tokio::time::timeout(Duration::from_secs(10), async {
13637 while !started.exists() {
13638 tokio::time::sleep(Duration::from_millis(20)).await;
13639 }
13640 }) => result.unwrap(),
13641 }
13642 assert_eq!(
13643 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13644 -1,
13645 "mechanical edits must hold the real lock"
13646 );
13647 std::fs::write(proceed, "ready").unwrap();
13648 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13649 .await
13650 .unwrap()
13651 .unwrap();
13652 drop(gate);
13653 assert!(outcomes.iter().all(CommandOutcome::ok));
13654 assert!(checked.exists());
13655 assert!(runner.state.pre_gate_commit.is_some());
13656 assert!(git::is_clean(&winner.worktree).await.unwrap());
13657 assert_eq!(
13658 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13659 0,
13660 "gate completion must release coordination"
13661 );
13662 drop(outsider);
13663 std::fs::remove_file(common.join("updater.lock")).unwrap();
13664 std::fs::create_dir(common.join("updater.lock")).unwrap();
13665 std::fs::remove_file(&started).unwrap();
13666 std::fs::remove_file(&checked).unwrap();
13667 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13668 let error = runner.run_gate(&winner).await.unwrap_err();
13669 assert!(
13670 error
13671 .to_string()
13672 .contains("acquire mandatory repository lock")
13673 );
13674 assert!(
13675 !started.exists(),
13676 "failed acquisition must prevent pre-gate"
13677 );
13678 assert!(!checked.exists(), "failed acquisition must prevent gate");
13679 assert!(git::is_clean(&winner.worktree).await.unwrap());
13680 assert_eq!(
13681 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13682 head
13683 );
13684 }
13685
13686 #[tokio::test]
13687 #[cfg(unix)]
13688 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13689 use std::os::fd::AsRawFd as _;
13690 use std::time::Duration;
13691
13692 ask_test_home();
13693 let tmp = tempfile::tempdir().unwrap();
13694 let repo = tmp.path().join("repo");
13695 std::fs::create_dir_all(&repo).unwrap();
13696 init_repo(&repo);
13697 let common = repo.join(".git");
13698 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13699 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13700 assert_eq!(
13701 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13702 0
13703 );
13704 let mut runner = runner_at(RunStatus::Prep);
13705 runner.state.repo = repo.clone();
13706 runner.state.config.coordination = crate::config::test_coordination();
13707 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13708 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13709 runner.state.config.graph.advise = true;
13710 runner.state.config.graph.advisors = 1;
13711 runner.roles.implementers = vec![spec("implement")];
13712 runner.roles.judges = vec![spec("judge")];
13713 let root = runner.state.worktree_root();
13714 let started = tmp.path().join("setup-started");
13715 let proceed = tmp.path().join("setup-proceed");
13716 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13717 run: Some(format!(
13718 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13719 started.display(),
13720 proceed.display()
13721 )),
13722 ..Default::default()
13723 }];
13724 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13725 let config = git::git(&repo, &["config", "--local", "--list"])
13726 .await
13727 .unwrap();
13728 let mut prep = Box::pin(runner.prep());
13729 assert!(
13730 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13731 .await
13732 .is_err()
13733 );
13734 assert!(
13735 !root.exists(),
13736 "contention must prevent even checkout creation"
13737 );
13738 assert!(!started.exists(), "contention must prevent setup");
13739 assert_eq!(
13740 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13741 branches
13742 );
13743 assert_eq!(
13744 git::git(&repo, &["config", "--local", "--list"])
13745 .await
13746 .unwrap(),
13747 config
13748 );
13749 drop(lock);
13750 tokio::select! {
13751 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13752 result = tokio::time::timeout(Duration::from_secs(10), async {
13753 while !started.exists() {
13754 tokio::time::sleep(Duration::from_millis(20)).await;
13755 }
13756 }) => result.unwrap(),
13757 }
13758 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13759 assert_eq!(
13760 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13761 -1
13762 );
13763 assert!(
13764 tokio::time::timeout(
13765 Duration::from_millis(100),
13766 crate::repo_lock::SeatLease::for_checkout(
13767 &repo,
13768 true,
13769 crate::config::test_coordination().as_ref()
13770 )
13771 )
13772 .await
13773 .is_err(),
13774 "preparation must also exclude local editors"
13775 );
13776 std::fs::write(proceed, "ready").unwrap();
13777 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13778 .await
13779 .unwrap()
13780 .unwrap();
13781 drop(prep);
13782 assert_eq!(runner.state.prep_complete, Some(true));
13783 assert!(root.join("judge-1").exists());
13784 assert!(root.join("advisor-1").exists());
13785 assert_eq!(
13786 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13787 0
13788 );
13789 }
13790
13791 #[tokio::test]
13792 #[cfg(unix)]
13793 async fn preparation_lock_failure_prevents_repository_changes() {
13794 ask_test_home();
13795 let tmp = tempfile::tempdir().unwrap();
13796 let repo = tmp.path().join("repo");
13797 std::fs::create_dir_all(&repo).unwrap();
13798 init_repo(&repo);
13799 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13800 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13803 let mut runner = runner_at(RunStatus::Prep);
13804 runner.state.repo = repo.clone();
13805 runner.state.config.coordination = crate::config::test_coordination();
13806 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13807 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13808 runner.roles.implementers = vec![spec("implement")];
13809 let error = runner.prep().await.unwrap_err();
13810 assert!(
13811 error
13812 .to_string()
13813 .contains("acquire mandatory repository lock")
13814 );
13815 assert!(!runner.state.worktree_root().exists());
13816 assert!(runner.state.candidates.is_empty());
13817 assert!(!repo.join(".git/config.worktree").exists());
13818 assert_eq!(
13819 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13820 "* main"
13821 );
13822 }
13823
13824 #[tokio::test]
13825 async fn preparation_records_creation_intent_before_git_can_fail() {
13826 ask_test_home();
13827 let tmp = tempfile::tempdir().unwrap();
13828 let repo = tmp.path().join("repo");
13829 std::fs::create_dir_all(&repo).unwrap();
13830 init_repo(&repo);
13831 let mut runner = runner_at(RunStatus::Prep);
13832 runner.state.repo = repo.clone();
13833 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13834 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13835 runner.state.config.blind.commit_msg_hook = false;
13836 runner.roles.implementers = vec![spec("one")];
13837 let label = blind::assign_labels(1, runner.state.seed)[0];
13838 let branch = runner.state.branch_for(label);
13839 git::git(&repo, &["branch", &branch]).await.unwrap();
13842 assert!(runner.prep().await.is_err());
13843 let saved = RunState::load(&runner.state.id).unwrap();
13844 assert!(saved.candidates.is_empty());
13845 let event = saved
13846 .events
13847 .iter()
13848 .find(|e| e.node == "prep-intent")
13849 .unwrap();
13850 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13851 assert_eq!(intent["branch"], branch);
13852 assert_eq!(intent["start"], runner.state.base_commit);
13853 assert_eq!(
13854 intent["worktree"],
13855 runner
13856 .state
13857 .worktree_root()
13858 .join(format!("cand-{label}"))
13859 .to_string_lossy()
13860 .as_ref()
13861 );
13862 assert_eq!(intent["picks"], serde_json::json!([]));
13863 }
13864
13865 #[tokio::test]
13866 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13867 ask_test_home();
13868 let tmp = tempfile::tempdir().unwrap();
13869 let repo = tmp.path().join("repo");
13870 std::fs::create_dir_all(&repo).unwrap();
13871 init_repo(&repo);
13872 let mut runner = runner_at(RunStatus::Prep);
13873 runner.state.repo = repo.clone();
13874 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13875 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13876 runner.state.sources.pick_commits.push(refs::Input {
13877 token: "missing".into(),
13878 sha: "0".repeat(40),
13879 provenance: refs::Provenance::ExplicitCli,
13880 });
13881 assert!(runner.prep().await.is_err());
13882 assert!(runner.state.candidates.is_empty());
13883 assert!(!runner.state.worktree_root().exists());
13884 }
13885
13886 #[tokio::test]
13887 async fn prose_and_legacy_inference_never_import_into_candidates() {
13888 ask_test_home();
13889 let tmp = tempfile::tempdir().unwrap();
13890 let repo = tmp.path().join("repo");
13891 std::fs::create_dir_all(&repo).unwrap();
13892 init_repo(&repo);
13893 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13894 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13895 .await
13896 .unwrap();
13897 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13898 git::git(&repo, &["add", "."]).await.unwrap();
13899 git::git(&repo, &["commit", "-m", "historical implementation"])
13900 .await
13901 .unwrap();
13902 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13903 git::git(&repo, &["checkout", "main"]).await.unwrap();
13904 for instruction in [
13905 format!("Example: magi/history/A at {source}"),
13906 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13907 format!("Never import magi/history/A or {source}"),
13908 format!("Conversation answer: I mentioned {source} only as background"),
13909 ] {
13910 let mut runner = runner_at(RunStatus::Prep);
13911 runner.state.repo = repo.clone();
13912 runner.state.base_commit = base.clone();
13913 runner.state.instruction = instruction.clone();
13914 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13915 runner.state.config.blind.commit_msg_hook = false;
13916 runner.state.seeds = vec![refs::Seed {
13917 token: "magi/history/A".into(),
13918 sha: source.clone(),
13919 kind: refs::SeedKind::Unmerged,
13920 branch: true,
13921 detail: String::new(),
13922 }];
13923 runner.roles.implementers = vec![spec("one")];
13924 assert_eq!(seeded_instruction(&runner.state), instruction);
13925 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13927 legacy.as_object_mut().unwrap().remove("sources");
13928 runner.state = serde_json::from_value(legacy).unwrap();
13929 assert_eq!(runner.state.sources, refs::Sources::default());
13930 runner.state.save().unwrap();
13931 runner.state = RunState::load(&runner.state.id).unwrap();
13932 runner.prep().await.unwrap();
13933 for candidate in &runner.state.candidates {
13934 assert_eq!(
13935 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13936 base
13937 );
13938 assert!(!candidate.worktree.join("unwanted").exists());
13939 }
13940 }
13941 }
13942
13943 #[tokio::test]
13944 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13945 ask_test_home();
13946 let tmp = tempfile::tempdir().unwrap();
13947 let repo = tmp.path().join("repo");
13948 std::fs::create_dir_all(&repo).unwrap();
13949 init_repo(&repo);
13950 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13951 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13952 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13953 git::git(&repo, &["add", "."]).await.unwrap();
13954 git::git(&repo, &["commit", "-m", "required seed"])
13955 .await
13956 .unwrap();
13957 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13958 git::git(&repo, &["checkout", "main"]).await.unwrap();
13959 let mut runner = runner_at(RunStatus::Prep);
13960 runner.state.repo = repo.clone();
13961 runner.state.base_commit = base.clone();
13962 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13963 runner.state.config.graph.advise = false;
13964 runner.state.config.blind.commit_msg_hook = false;
13965 runner.state.sources =
13966 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13967 .await
13968 .unwrap();
13969 runner.roles.implementers = vec![spec("one"), spec("two")];
13970 runner.roles.judges = vec![spec("judge")];
13971 let label = blind::assign_labels(2, runner.state.seed)[0];
13972 let branch = runner.state.branch_for(label);
13973 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13974 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13975 .await
13976 .unwrap();
13977 runner.state.candidates.push(Candidate {
13978 index: 0,
13979 label,
13980 agent: "one".into(),
13981 branch,
13982 worktree,
13983 summary: String::new(),
13984 stat: String::new(),
13985 files: 0,
13986 commits: 0,
13987 empty: false,
13988 failed: None,
13989 verified_noop: None,
13990 duration_ms: 0,
13991 folded: false,
13992 });
13993 runner.state.save().unwrap();
13994 runner.state = RunState::load(&runner.state.id).unwrap();
13996 assert_eq!(runner.state.prep_complete, Some(false));
13997 runner.prep().await.unwrap();
13998 assert_eq!(runner.state.prep_complete, Some(true));
13999 assert_eq!(runner.state.candidates.len(), 2);
14000 for candidate in &runner.state.candidates {
14001 assert_eq!(
14002 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14003 git::tree_of(&repo, &seed).await.unwrap()
14004 );
14005 assert_eq!(
14006 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14007 "required commit"
14008 );
14009 }
14010 assert!(runner.state.worktree_root().join("judge-1").exists());
14011 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14012 runner.prep().await.unwrap();
14013 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14014
14015 runner.state.prep_complete = Some(false);
14016 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14017 run: Some(format!(
14018 "test -f '{}'",
14019 tmp.path().join("dependency-ready").display()
14020 )),
14021 ..Default::default()
14022 }];
14023 assert!(runner.prep().await.is_err());
14024 assert!(runner.state.prep_failure.is_some());
14025 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14026 runner.state = RunState::load(&runner.state.id).unwrap();
14027 assert!(
14028 runner.prep().await.is_err(),
14029 "correcting the cause alone must not auto-retry"
14030 );
14031 let failed_at = runner.state.events.last().unwrap().at;
14032 assert!(
14033 runner.retry_preparation(failed_at).is_err(),
14034 "old approvals cannot unlock the hold"
14035 );
14036 let mut task = crate::queue::Task::new(
14037 "retry".into(),
14038 "retry setup".into(),
14039 repo.clone(),
14040 crate::queue::Source::Human,
14041 );
14042 task.start(runner.state.id.clone());
14043 task.hold_machine(Some("setup failed".into()));
14044 task.release_operator("cli", None);
14045 runner
14046 .retry_preparation(task.resumption.as_ref().unwrap().at)
14047 .unwrap();
14048 runner.prep().await.unwrap();
14049 assert!(runner.state.prep_failure.is_none());
14050 assert_eq!(runner.state.prep_complete, Some(true));
14051 assert_eq!(runner.state.candidates.len(), 2);
14052 assert!(runner.state.worktree_root().join("judge-1").exists());
14053 for candidate in &runner.state.candidates {
14054 assert_eq!(
14055 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14056 git::tree_of(&repo, &seed).await.unwrap()
14057 );
14058 }
14059 }
14060
14061 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14062 let mut heads = Vec::new();
14063 for candidate in candidates {
14064 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14065 }
14066 heads
14067 }
14068
14069 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14070 ask_test_home();
14071 let tmp = tempfile::tempdir().unwrap();
14072 let repo = tmp.path().join("repo");
14073 std::fs::create_dir(&repo).unwrap();
14074 init_repo(&repo);
14075 let mut runner = runner_at(RunStatus::Reviewing);
14076 runner.state.repo = repo.clone();
14077 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14078 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14079 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14080 let branch = format!("candidate-{label}");
14081 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14082 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14083 .await
14084 .unwrap();
14085 if label == 'B' {
14086 std::fs::write(worktree.join("change"), "winner").unwrap();
14087 git::rescue_commit(&worktree, "candidate change")
14088 .await
14089 .unwrap();
14090 }
14091 runner.state.candidates.push(Candidate {
14092 index,
14093 label,
14094 agent: "alpha".into(),
14095 branch,
14096 worktree,
14097 summary: "completed reply".into(),
14098 stat: String::new(),
14099 files: usize::from(label == 'B'),
14100 commits: usize::from(label == 'B'),
14101 empty: label == 'A',
14102 failed: None,
14103 verified_noop: None,
14104 duration_ms: 1,
14105 folded: false,
14106 });
14107 }
14108 runner.state.judge_skipped = true;
14109 runner.state.tally = Some(Tally {
14110 first_choice: BTreeMap::new(),
14111 borda: BTreeMap::new(),
14112 winner: 'B',
14113 rankings: 0,
14114 unanimous_initial: false,
14115 deliberated: false,
14116 changed_votes: 0,
14117 unanimous_final: false,
14118 tie_break: None,
14119 judges: 0,
14120 present: 0,
14121 quorum: 0,
14122 met_quorum: true,
14123 uncontested: Some("only B produced a change".into()),
14124 });
14125 runner.state.save().unwrap();
14126 (tmp, runner)
14127 }
14128
14129 #[tokio::test]
14130 #[cfg(unix)]
14131 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14132 for (solo, failed_checkpoint) in
14133 [(true, false), (false, false), (true, true), (false, true)]
14134 {
14135 let (_tmp, mut runner) = preservation_resume_fixture().await;
14136 if solo {
14137 runner.state.candidates.retain(|c| c.label == 'A');
14138 }
14139 runner.state.candidates[0].empty = false;
14140 runner.state.candidates[0].summary.clear();
14141 runner.state.tally = None;
14142 runner.state.judge_skipped = false;
14143 runner.state.status = RunStatus::Implementing;
14144 runner.state.config.disk.min_free_bytes = u64::MAX;
14145 let mut implementer = spec("alpha");
14146 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14147 runner.roles.implementers = vec![implementer];
14148 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14149 if failed_checkpoint {
14150 std::fs::create_dir_all(&patch).unwrap();
14151 }
14152 runner.state.save().unwrap();
14153 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14154 runner.state = RunState::load(&runner.state.id).unwrap();
14155 assert_eq!(runner.state.status, RunStatus::Blocked);
14156 assert!(runner.state.interrupted_implementations.contains(&'A'));
14157 assert_eq!(runner.state.candidates[0].commits, 0);
14158 assert!(!runner.state.candidates[0].empty);
14159 assert!(runner.state.candidates[0].failed.is_none());
14160 if failed_checkpoint {
14161 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14162 std::fs::remove_dir(patch).unwrap();
14163 }
14164 runner.state.resource_block = None;
14165 runner.state.config.disk.min_free_bytes = 0;
14166 runner.roles.implementers[0].command = vec![
14167 "sh".into(),
14168 "-c".into(),
14169 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14170 ];
14171 runner.implement().await.unwrap();
14172 assert!(runner.state.interrupted_implementations.is_empty());
14173 assert!(runner.state.candidate_preservation.is_empty());
14174 assert_eq!(runner.state.candidates[0].commits, 1);
14175 assert!(!runner.state.candidates[0].empty);
14176 assert!(
14177 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14178 .unwrap()
14179 .contains("retried empty interrupted seat")
14180 );
14181 if !solo {
14182 assert_eq!(runner.state.candidates[1].commits, 1);
14183 }
14184 }
14185 }
14186
14187 #[tokio::test]
14188 #[cfg(unix)]
14189 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14190 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14191 let (_tmp, mut runner) = preservation_resume_fixture().await;
14192 runner.state.candidates.retain(|c| c.label == 'A');
14193 runner.state.candidates[0].empty = false;
14194 runner.state.candidates[0].summary.clear();
14195 runner.state.tally = None;
14196 runner.state.judge_skipped = false;
14197 runner.state.status = RunStatus::Implementing;
14198 runner.state.config.disk.min_free_bytes = u64::MAX;
14201 let wt = runner.state.candidates[0].worktree.clone();
14202 std::fs::write(
14203 wt.join("interrupted-source"),
14204 "retain implementation work\n",
14205 )
14206 .unwrap();
14207 if committed {
14208 git::rescue_commit(&wt, "implementation checkpoint")
14209 .await
14210 .unwrap();
14211 }
14212 let lock = git::git(
14213 &wt,
14214 &[
14215 "rev-parse",
14216 "--path-format=absolute",
14217 "--git-path",
14218 "index.lock",
14219 ],
14220 )
14221 .await
14222 .unwrap();
14223 if block_rescue {
14224 std::fs::write(&lock, "fixture-owned lock").unwrap();
14225 }
14226 let mut implementer = spec("alpha");
14227 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14228 runner.roles.implementers = vec![implementer];
14229 runner.state.save().unwrap();
14230 let result = runner.implement().await;
14231 assert_eq!(result.is_err(), block_rescue);
14232 assert_eq!(
14233 runner.state.resource_block.as_ref().unwrap().kind,
14234 "disk pressure"
14235 );
14236 assert!(
14237 runner
14238 .state
14239 .events
14240 .iter()
14241 .any(|e| e.message.contains("agent execution interrupted"))
14242 );
14243 assert!(
14244 runner
14245 .state
14246 .dir()
14247 .join("artifacts/candidate-A-recovery/staged.patch")
14248 .exists()
14249 );
14250 if block_rescue {
14251 let saved = RunState::load(&runner.state.id).unwrap();
14252 assert!(saved.candidate_preservation.contains_key(&'A'));
14253 assert!(!saved.candidates[0].empty);
14254 assert_eq!(
14255 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14256 "retain implementation work\n"
14257 );
14258 std::fs::remove_file(lock).unwrap();
14259 runner.state = saved;
14260 } else {
14261 assert_eq!(runner.state.status, RunStatus::Blocked);
14262 assert_eq!(runner.state.candidates[0].commits, 1);
14263 }
14264 runner.roles.implementers.clear();
14267 runner.state.config.disk.min_free_bytes = 0;
14268 runner.state.resource_block = None;
14269 runner.implement().await.unwrap();
14270 assert!(runner.state.candidate_preservation.is_empty());
14271 assert_eq!(runner.state.candidates[0].commits, 1);
14272 assert!(!runner.state.candidates[0].empty);
14273 assert!(git::is_clean(&wt).await.unwrap());
14274 assert!(
14275 crate::run::read_artifact(&runner.state, "cand-A.patch")
14276 .unwrap()
14277 .contains("retain implementation work")
14278 );
14279 }
14280 }
14281
14282 #[tokio::test]
14283 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14284 let (_tmp, mut runner) = preservation_resume_fixture().await;
14285 runner.implement().await.unwrap();
14287 assert!(runner.state.candidate_preservation.is_empty());
14288 assert!(
14289 !runner
14290 .state
14291 .dir()
14292 .join("artifacts/candidate-A-recovery")
14293 .exists()
14294 );
14295 runner.fold_losers().await.unwrap();
14296 assert!(!runner.state.candidates[0].worktree.exists());
14297 for schema in [14, crate::run::SCHEMA] {
14298 runner.state.schema = schema;
14299 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14300 runner.state.save().unwrap();
14301 runner.state = RunState::load(&runner.state.id).unwrap();
14302 runner.implement().await.unwrap();
14303 assert!(runner.state.candidate_preservation.is_empty());
14304 assert!(runner.state.candidates[0].folded);
14305 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14306 }
14307 }
14308
14309 #[tokio::test]
14310 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14311 let (_tmp, mut runner) = preservation_resume_fixture().await;
14312 let wt = runner.state.candidates[0].worktree.clone();
14313 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14314 runner.state.schema = 14;
14315 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14316 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14317 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14318 for _ in 0..2 {
14319 assert!(runner.implement().await.is_err());
14320 assert_eq!(runner.state.status, RunStatus::Blocked);
14321 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14322 assert!(
14323 runner.state.candidates[0].empty,
14324 "old verdict must remain unusable"
14325 );
14326 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14327 assert!(runner.state.judge_skipped);
14328 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14329 assert!(
14330 crate::run::read_artifact(&runner.state, "cand-A.patch")
14331 .unwrap()
14332 .contains("owned edits")
14333 );
14334 assert!(wt.exists());
14335 assert_eq!(
14336 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14337 .await
14338 .unwrap(),
14339 1
14340 );
14341 runner.state = RunState::load(&runner.state.id).unwrap();
14342 }
14343 }
14344
14345 #[tokio::test]
14346 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14347 let (_tmp, mut runner) = preservation_resume_fixture().await;
14348 let wt = runner.state.candidates[0].worktree.clone();
14349 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14350 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14351 let index_lock = PathBuf::from(
14352 git::git(
14353 &wt,
14354 &[
14355 "rev-parse",
14356 "--path-format=absolute",
14357 "--git-path",
14358 "index.lock",
14359 ],
14360 )
14361 .await
14362 .unwrap(),
14363 );
14364 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14365 runner.state.schema = 14;
14366 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14367 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14368 assert!(runner.implement().await.is_err());
14369 assert!(runner.after_implement().is_err());
14370 assert!(runner.fold_losers().await.is_err());
14371 assert_eq!(
14372 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14373 "failed seat edits"
14374 );
14375 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14376 runner.state = RunState::load(&runner.state.id).unwrap();
14377 std::fs::remove_file(index_lock).unwrap();
14378 assert!(runner.implement().await.is_err());
14380 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14381 assert!(runner.state.candidates[0].empty);
14382 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14383 assert!(
14384 crate::run::read_artifact(&runner.state, "cand-A.patch")
14385 .unwrap()
14386 .contains("failed seat edits")
14387 );
14388 assert!(wt.exists());
14389 }
14390
14391 #[tokio::test]
14392 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14393 let (_tmp, mut runner) = preservation_resume_fixture().await;
14394 let repo = runner.state.repo.clone();
14395 let common = git::git(
14396 &repo,
14397 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14398 )
14399 .await
14400 .unwrap();
14401 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14402 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14403 assert!(runner.fold_losers().await.is_err());
14404 assert!(!runner.state.candidates[0].worktree.exists());
14405 assert!(!runner.state.candidates[0].folded);
14406 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14407 runner.state = RunState::load(&runner.state.id).unwrap();
14408 std::fs::remove_file(branch_lock).unwrap();
14409 let judge = runner.state.worktree_root().join("judge-1");
14410 git::worktree_add_detached(&repo, &judge, "main")
14411 .await
14412 .unwrap();
14413 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14414 runner.roles.judges = vec![spec("judge")];
14415 assert!(runner.fold_losers().await.is_err());
14416 runner.state = RunState::load(&runner.state.id).unwrap();
14417 assert!(
14418 runner.state.candidates[0].folded,
14419 "candidate progress saved before judge failure"
14420 );
14421 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14422 assert!(judge.join("uncommitted").exists());
14423 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14424 runner.fold_losers().await.unwrap();
14425 assert!(!judge.exists());
14426 runner.fold_losers().await.unwrap();
14427 }
14428
14429 #[tokio::test]
14430 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14431 let (_tmp, mut runner) = preservation_resume_fixture().await;
14432 runner.state.candidates.retain(|c| c.label == 'B');
14433 runner.state.tally = None;
14434 runner.state.judge_skipped = false;
14435 runner.state.prep_complete = Some(true);
14436 runner.state.config.graph.advise = false;
14437 runner.state.config.graph.selected_implementer = Some("alpha".into());
14438 let wt = runner.state.candidates[0].worktree.clone();
14439 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14440 runner
14441 .state
14442 .candidate_preservation
14443 .insert('B', "awaiting persistence".into());
14444 let index_lock = git::git(
14445 &wt,
14446 &[
14447 "rev-parse",
14448 "--path-format=absolute",
14449 "--git-path",
14450 "index.lock",
14451 ],
14452 )
14453 .await
14454 .unwrap();
14455 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14456 runner.state.save().unwrap();
14457 assert!(runner.execute_graph().await.is_err());
14458 assert_eq!(runner.state.status, RunStatus::Blocked);
14459 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14460 std::fs::remove_file(index_lock).unwrap();
14461 for _ in 0..2 {
14464 runner.state = RunState::load(&runner.state.id).unwrap();
14465 runner.pause.park();
14466 runner.execute_graph().await.unwrap();
14467 assert_eq!(runner.state.status, RunStatus::Implementing);
14468 assert!(runner.state.candidate_preservation.is_empty());
14469 assert_eq!(runner.state.candidates[0].commits, 2);
14470 assert!(runner.state.tally.is_none());
14471 }
14472 runner.implement().await.unwrap();
14473 assert_eq!(runner.state.status, RunStatus::Judging);
14474 runner.state.status = RunStatus::Blocked;
14476 runner.state.candidates[0].failed = Some("implementation failed".into());
14477 runner.state.save().unwrap();
14478 let error = runner.execute_graph().await.unwrap_err();
14479 assert!(
14480 error
14481 .to_string()
14482 .contains("selected implementation is held")
14483 );
14484 }
14485
14486 #[tokio::test]
14487 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14488 let (_tmp, mut runner) = preservation_resume_fixture().await;
14489 let repo = runner.state.repo.clone();
14490 let candidate = runner.state.candidates[0].clone();
14491 let registered = git::worktree_holding(&repo, &candidate.branch)
14494 .await
14495 .unwrap()
14496 .unwrap();
14497 assert_eq!(
14498 registered.canonicalize().unwrap(),
14499 candidate.worktree.canonicalize().unwrap()
14500 );
14501 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14502 assert_eq!(
14503 git::worktree_holding(&repo, &candidate.branch)
14504 .await
14505 .unwrap(),
14506 Some(registered)
14507 );
14508 assert!(
14509 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14510 "the stale registration initially keeps the branch checked out"
14511 );
14512 runner.fold_losers().await.unwrap();
14513 assert!(runner.state.candidates[0].folded);
14514 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14515 assert_eq!(
14516 git::worktree_holding(&repo, &candidate.branch)
14517 .await
14518 .unwrap(),
14519 None
14520 );
14521 runner.state = RunState::load(&runner.state.id).unwrap();
14522 runner.fold_losers().await.unwrap();
14523 }
14524
14525 #[tokio::test]
14526 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14527 let (_tmp, mut runner) = preservation_resume_fixture().await;
14528 let repo = runner.state.repo.clone();
14529 let candidate = runner.state.candidates[0].clone();
14530 let winner = runner.state.candidates[1].clone();
14531 let registered = git::worktree_holding(&repo, &candidate.branch)
14532 .await
14533 .unwrap()
14534 .unwrap();
14535 assert_eq!(
14536 registered.canonicalize().unwrap(),
14537 candidate.worktree.canonicalize().unwrap()
14538 );
14539 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14540 assert_eq!(
14541 git::worktree_holding(&repo, &candidate.branch)
14542 .await
14543 .unwrap(),
14544 Some(registered)
14545 );
14546 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14547
14548 for _ in 0..2 {
14549 fold_run(&mut runner.state, false, &crate::run::home())
14550 .await
14551 .unwrap();
14552 runner.state = RunState::load(&runner.state.id).unwrap();
14553 assert!(runner.state.candidates[0].folded);
14554 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14555 assert_eq!(
14556 git::worktree_holding(&repo, &candidate.branch)
14557 .await
14558 .unwrap(),
14559 None
14560 );
14561 assert!(!runner.state.candidates[1].folded);
14562 assert!(winner.worktree.exists());
14563 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14564 }
14565 }
14566
14567 #[tokio::test]
14568 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14569 ask_test_home();
14570 let tmp = tempfile::tempdir().unwrap();
14571 let repo = tmp.path().join("repo");
14572 std::fs::create_dir(&repo).unwrap();
14573 init_repo(&repo);
14574 let mut runner = runner_at(RunStatus::Implementing);
14575 runner.state.repo = repo.clone();
14576 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14577 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14578 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14579 let branch = format!("candidate-{label}");
14580 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14581 git::worktree_add_branch(&repo, &wt, &branch, "main")
14582 .await
14583 .unwrap();
14584 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14585 git::git(&wt, &["add", "README.md"]).await.unwrap();
14586 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14587 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14588 runner.state.candidates.push(Candidate {
14589 index,
14590 label,
14591 agent: "alpha".to_owned(),
14592 branch,
14593 worktree: wt,
14594 summary: "completed implementation".to_owned(),
14595 stat: String::new(),
14596 files: 0,
14597 commits: 0,
14598 empty: false,
14599 failed: None,
14600 verified_noop: None,
14601 duration_ms: 42,
14602 folded: false,
14603 });
14604 runner
14605 .state
14606 .candidate_preservation
14607 .insert(label, "awaiting persistence".to_owned());
14608 }
14609 let wt = runner.state.candidates[0].worktree.clone();
14610 let index_lock = git::git(
14611 &wt,
14612 &[
14613 "rev-parse",
14614 "--path-format=absolute",
14615 "--git-path",
14616 "index.lock",
14617 ],
14618 )
14619 .await
14620 .unwrap();
14621 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14622 runner.state.save().unwrap();
14623 assert!(runner.persist_candidates().await.is_err());
14624 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14625 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14626 assert_eq!(
14627 runner.state.candidates[1].commits, 1,
14628 "other results are still collected"
14629 );
14630 assert!(
14631 !runner.state.candidates[0].empty,
14632 "failed rescue is not no change"
14633 );
14634 assert_eq!(
14635 runner.state.candidates[0].summary,
14636 "completed implementation"
14637 );
14638 assert!(runner.after_implement().is_err());
14639 let home = tmp.path().join("fold-home");
14640 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14641 assert!(wt.join("new-file").exists());
14642 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14643 assert!(recovery.join("staged.patch").exists());
14644 assert!(recovery.join("unstaged.patch").exists());
14645 assert_eq!(
14646 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14647 "untracked edits\n"
14648 );
14649 runner.state.begin_pass();
14650 runner.state.end_pass_parked(ParkCause::Upgrade {
14651 from: "1.0".into(),
14652 to: Some("1.1".into()),
14653 });
14654 runner.state.save().unwrap();
14655 runner.state = RunState::load(&runner.state.id).unwrap();
14656 assert!(matches!(
14657 runner.state.passes.last().unwrap().end,
14658 Some(crate::run::PassEnd::Parked {
14659 cause: ParkCause::Upgrade { .. }
14660 })
14661 ));
14662 std::fs::remove_file(index_lock).unwrap();
14663 runner.implement().await.unwrap();
14665 assert!(runner.state.candidate_preservation.is_empty());
14666 assert!(
14667 runner
14668 .state
14669 .candidates
14670 .iter()
14671 .all(|c| c.commits == 1 && !c.empty)
14672 );
14673 assert!(git::is_clean(&wt).await.unwrap());
14674 assert!(
14675 crate::run::read_artifact(&runner.state, "cand-A.patch")
14676 .unwrap()
14677 .contains("later tracked edits")
14678 );
14679 }
14680
14681 #[tokio::test]
14682 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14683 ask_test_home();
14684 let tmp = tempfile::tempdir().unwrap();
14685 let repo = tmp.path().join("repo");
14686 std::fs::create_dir(&repo).unwrap();
14687 init_repo(&repo);
14688 let wt = tmp.path().join("candidate");
14689 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14690 .await
14691 .unwrap();
14692 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14693 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14694 assert!(wt.join("uncommitted").exists());
14695 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14696 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14697 let mut runner = runner_at(RunStatus::Implementing);
14698 runner.state.repo = repo.clone();
14699 runner.state.base_commit = "missing-base".to_owned();
14700 runner.state.candidates.push(Candidate {
14701 index: 0,
14702 label: 'A',
14703 agent: "alpha".into(),
14704 branch: "candidate".into(),
14705 worktree: wt.clone(),
14706 summary: "completed".into(),
14707 stat: String::new(),
14708 files: 0,
14709 commits: 0,
14710 empty: false,
14711 failed: None,
14712 verified_noop: None,
14713 duration_ms: 1,
14714 folded: false,
14715 });
14716 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14717 assert!(
14718 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14719 .await
14720 .is_err()
14721 );
14722 assert!(!runner.state.candidates[0].folded);
14723 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14724 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14725 runner
14726 .state
14727 .candidate_preservation
14728 .insert('A', "pending".into());
14729 assert!(runner.persist_candidates().await.is_err());
14730 assert!(!runner.state.candidates[0].empty);
14731 assert!(!runner.state.candidates[0].folded);
14732 assert!(wt.exists());
14733 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14734 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14735 std::fs::create_dir_all(&patch_path).unwrap();
14736 assert!(
14737 runner.persist_candidates().await.is_err(),
14738 "artifact failure stays pending"
14739 );
14740 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14741 assert!(!runner.state.candidates[0].empty);
14742 std::fs::remove_dir(patch_path).unwrap();
14743 runner.persist_candidates().await.unwrap();
14744 assert!(
14745 runner.state.candidates[0].empty,
14746 "only successful inspection confirms no change"
14747 );
14748 }
14749
14750 fn init_repo(dir: &Path) {
14751 let run = |args: &[&str]| {
14752 let out = std::process::Command::new("git")
14753 .args(args)
14754 .current_dir(dir)
14755 .quiet()
14756 .output()
14757 .expect("spawn git");
14758 assert!(
14759 out.status.success(),
14760 "git {args:?} failed: {}",
14761 String::from_utf8_lossy(&out.stderr)
14762 );
14763 };
14764 run(&["init", "-b", "main"]);
14765 run(&["config", "user.name", "magi test"]);
14766 run(&["config", "user.email", "magi@example.com"]);
14767 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14768 run(&["add", "-A"]);
14769 run(&["commit", "-m", "init"]);
14770 }
14771
14772 fn ask_test_home() {
14780 crate::run::pin_test_home();
14781 }
14782
14783 fn runner_at(status: RunStatus) -> Runner {
14786 let mut state = RunState::new(
14787 PathBuf::from("/nonexistent/repo"),
14788 "main".to_owned(),
14789 "deadbeef".to_owned(),
14790 "task".to_owned(),
14791 Config::default(),
14792 );
14793 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14796 state.id = format!(
14797 "20000101-000000-{:04x}",
14798 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14799 );
14800 state.status = status;
14801 Runner {
14802 state,
14803 roles: ResolvedRoles {
14804 implementers: Vec::new(),
14805 judges: Vec::new(),
14806 reviewers: Vec::new(),
14807 fixer: None,
14808 conductor: conductor(),
14809 implementer_roster: Vec::new(),
14810 judge_roster: Vec::new(),
14811 reviewer_roster: Vec::new(),
14812 },
14813 sem: Arc::new(Semaphore::new(1)),
14814 pause: Pause::new(),
14815 interrupt: Pause::new(),
14816 }
14817 }
14818
14819 #[tokio::test]
14820 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14821 crate::run::pin_test_home();
14822 let mut runner = runner_at(RunStatus::Gating);
14823 runner.state.save().unwrap();
14824 let id = runner.state.id.clone();
14825 let passes = runner.state.passes.len();
14826 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14827 let mut claiming = Box::pin(runner.claim_driver());
14828 std::future::poll_fn(|cx| {
14829 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14830 std::task::Poll::Ready(())
14831 })
14832 .await;
14833 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14834 drop(owner);
14835 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14836 .await
14837 .unwrap()
14838 .unwrap();
14839 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14840 assert!(execution.runner.state.resource_block.is_none());
14841 assert_eq!(
14842 execution.runner.state.liveness(false),
14843 crate::run::Liveness::Live
14844 );
14845 drop(execution);
14846 assert!(RunState::load(&id).unwrap().driver_exited);
14847 }
14848
14849 #[tokio::test]
14850 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14851 crate::run::pin_test_home();
14852 let mut runner = runner_at(RunStatus::Gating);
14853 runner.state.save().unwrap();
14854 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14857 assert!(runner.claim_driver().await.is_err());
14858 assert!(runner.state.driver_exited);
14859 assert_eq!(runner.state.driver_pid, None);
14860 let disk = RunState::load(&runner.state.id).unwrap();
14861 assert!(disk.driver_exited);
14862 assert_eq!(disk.driver_pid, None);
14863 assert_eq!(disk.status, RunStatus::Gating);
14864 }
14865
14866 #[tokio::test]
14867 async fn an_early_graph_error_releases_its_driver() {
14868 crate::run::pin_test_home();
14869 let mut runner = runner_at(RunStatus::Blocked);
14870 runner.state.config.graph.selected_implementer = Some("selected".into());
14871 runner.state.save().unwrap();
14872 assert!(runner.execute().await.is_err());
14873 let disk = RunState::load(&runner.state.id).unwrap();
14874 assert!(disk.driver_exited);
14875 assert_eq!(disk.driver_pid, None);
14876 assert_eq!(disk.status, RunStatus::Blocked);
14877 }
14878
14879 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14880 crate::run::MergeRepair {
14881 question: "repair-question".into(),
14882 head: "old-head".into(),
14883 pr: "https://github.com/o/r/pull/42".into(),
14884 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14885 gate: vec![outcome(Some(0), false)],
14886 gate_head: Some("old-head".into()),
14887 gate_fixes: Vec::new(),
14888 pre_gate: Vec::new(),
14889 phase,
14890 failure: None,
14891 }
14892 }
14893
14894 #[tokio::test]
14895 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14896 crate::run::pin_test_home();
14897 let mut runner = runner_at(RunStatus::Landing);
14898 runner.state.merge_repairs.push(merge_repair_request(
14899 crate::run::MergeRepairPhase::Requested,
14900 ));
14901 runner.state.save().unwrap();
14902 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14903 assert!(
14904 runner
14905 .execute()
14906 .await
14907 .unwrap_err()
14908 .to_string()
14909 .contains("already running")
14910 );
14911 assert_eq!(
14912 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14913 crate::run::MergeRepairPhase::Requested
14914 );
14915 drop(claim);
14916 let mut disk = runner.state.clone();
14917 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14918 disk.save().unwrap();
14919 assert!(
14920 runner
14921 .execute()
14922 .await
14923 .unwrap_err()
14924 .to_string()
14925 .contains("decision changed")
14926 );
14927 assert_eq!(
14928 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14929 crate::run::MergeRepairPhase::Complete
14930 );
14931 }
14932
14933 #[test]
14934 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14935 crate::run::pin_test_home();
14936 let mut runner = runner_at(RunStatus::Landing);
14937 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14938 runner.state.reviews = request.reviews.clone();
14939 runner.state.gate = request.gate.clone();
14940 runner.state.gate_ran = true;
14941 runner.state.land_approval = Some(crate::run::LandApproval {
14942 question: "old".into(),
14943 head: "old-head".into(),
14944 });
14945 runner.state.merge_repairs.push(request);
14946 runner.begin_merge_repair_review(0).unwrap();
14947 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14948 assert!(!runner.state.gate_status().ok());
14949 assert!(runner.state.land_approval.is_none());
14950 let disk = RunState::load(&runner.state.id).unwrap();
14951 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14952 assert!(disk.merge_repairs[0].gate[0].ok());
14953 assert_eq!(
14954 disk.merge_repairs[0].phase,
14955 crate::run::MergeRepairPhase::Reviewing
14956 );
14957 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14958 assert!(disk.reviews.is_empty());
14959 }
14960
14961 #[tokio::test]
14962 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14963 crate::run::pin_test_home();
14964 for phase in [
14965 crate::run::MergeRepairPhase::Fixing,
14966 crate::run::MergeRepairPhase::Requested,
14967 ] {
14968 let mut runner = runner_at(RunStatus::Landing);
14969 runner.state.config.graph.land_rounds = 0;
14970 runner.state.merge_repairs.push(merge_repair_request(phase));
14971 runner.state.save().unwrap();
14972 assert!(!runner.repair_merge_review().await.unwrap());
14973 assert_eq!(runner.state.status, RunStatus::Blocked);
14974 assert_eq!(
14975 runner.state.merge_repairs[0].phase,
14976 crate::run::MergeRepairPhase::Failed
14977 );
14978 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14979 assert!(
14980 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14981 "interrupted"
14982 } else {
14983 "budget exhausted"
14984 })
14985 );
14986 assert!(runner.state.merge.is_none());
14987 }
14988 }
14989
14990 #[test]
14994 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14995 crate::run::pin_test_home();
14996 let mut runner = runner_at(RunStatus::Implementing);
14997 let interrupt = Pause::new();
14998 runner.watch_interrupt(interrupt.clone());
14999
15000 interrupt.park_because("task a1b2 asked to run first");
15001
15002 assert!(runner.park_here().expect("park_here"));
15003 assert!(runner.state.parked);
15004 let last = runner.state.events.last().expect("a park event");
15005 assert_eq!(last.node, "park");
15006 assert!(
15007 last.message.contains("task a1b2 asked to run first"),
15008 "expected the interrupt reason in {:?}",
15009 last.message
15010 );
15011 }
15012
15013 #[test]
15016 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15017 crate::run::pin_test_home();
15018 let mut runner = runner_at(RunStatus::Implementing);
15019 runner.state.begin_pass();
15020 let shutdown = Pause::new();
15021 runner.on_pause(shutdown.clone());
15022 let upgrade = ParkCause::Upgrade {
15023 from: "0.121.0".to_owned(),
15024 to: Some("0.121.1".to_owned()),
15025 };
15026 shutdown.park_for(upgrade.clone());
15027 shutdown.park_for(ParkCause::Operator);
15028
15029 assert!(runner.park_here().expect("park_here"));
15030 assert_eq!(
15031 runner.state.passes[0].end,
15032 Some(crate::run::PassEnd::Parked { cause: upgrade })
15033 );
15034 let last = runner.state.events.last().expect("a park event");
15035 assert!(last.message.contains("0.121.0"), "{}", last.message);
15036 }
15037
15038 #[tokio::test]
15039 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15040 crate::run::pin_test_home();
15041 let mut runner = runner_at(RunStatus::Reviewing);
15042 runner.state.save().unwrap();
15043 let mut execution = runner.claim_driver().await.expect("first execution");
15044 assert_eq!(execution.runner.state.passes.len(), 1);
15045 execution
15046 .driver
15047 .finish(&mut execution.runner.state)
15048 .await
15049 .unwrap();
15050 }
15051
15052 #[tokio::test]
15053 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15054 crate::run::pin_test_home();
15055 let mut runner = runner_at(RunStatus::Reviewing);
15056 runner.state.config.agents = vec![conductor()];
15057 runner.state.config.graph.timeout_resource_wait = 0;
15058 runner.state.save().unwrap();
15059 let id = runner.state.id.clone();
15060 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15061 assert!(runner.claim_driver().await.is_err());
15062 drop(owner);
15063 drop(runner);
15064 let mut resumed = Runner::resume(&id).unwrap();
15065 assert_eq!(resumed.state.driver_claimed, Some(false));
15066 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15067 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15068 let mut execution = resumed
15069 .claim_driver()
15070 .await
15071 .expect("retry initial claim after reload");
15072 assert_eq!(execution.runner.state.driver_claimed, Some(true));
15073 assert_eq!(execution.runner.state.passes.len(), 1);
15074 execution
15075 .driver
15076 .finish(&mut execution.runner.state)
15077 .await
15078 .unwrap();
15079 }
15080
15081 #[tokio::test]
15082 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15083 crate::run::pin_test_home();
15084 let mut runner = runner_at(RunStatus::Reviewing);
15085 runner.state.config.agents = vec![conductor()];
15086 runner.state.save().unwrap();
15087 let id = runner.state.id.clone();
15088 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15089 let mut resumed = Runner::resume(&id).unwrap();
15090 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15091 assert!(resumed.claim_driver().await.is_err());
15092 assert!(RunState::load(&id).unwrap().passes.is_empty());
15093 drop(lease);
15094 let mut execution = resumed.claim_driver().await.unwrap();
15095 execution
15096 .driver
15097 .finish(&mut execution.runner.state)
15098 .await
15099 .unwrap();
15100 }
15101
15102 #[tokio::test]
15103 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15104 crate::run::pin_test_home();
15105 let mut runner = runner_at(RunStatus::Reviewing);
15106 let state = &mut runner.state;
15107 state.config.agents = vec![conductor()];
15108 state.driver_claimed = None;
15109 state.driver_pid = Some(std::process::id());
15110 state.driver_started_at = None;
15111 state.begin_pass();
15112 state.save().unwrap();
15113 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15114 std::fs::write(
15115 state.dir().join("driver.json"),
15116 serde_json::to_vec(&serde_json::json!({
15117 "pid": state.driver_pid, "started_at": null,
15118 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15119 }))
15120 .unwrap(),
15121 )
15122 .unwrap();
15123 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15124 drop(lease);
15125 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15126 let mut resumed = Runner::resume(&state.id).unwrap();
15127 assert!(!resumed.state.driver_exited);
15128 let mut execution = resumed
15129 .claim_driver()
15130 .await
15131 .expect("lease proves previous execution ended");
15132 assert_eq!(execution.runner.state.passes.len(), 2);
15133 execution
15134 .driver
15135 .finish(&mut execution.runner.state)
15136 .await
15137 .unwrap();
15138 }
15139
15140 #[tokio::test]
15141 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15142 crate::run::pin_test_home();
15143 let mut runner = runner_at(RunStatus::Reviewing);
15144 let mut json = serde_json::to_value(&runner.state).unwrap();
15145 json.as_object_mut().unwrap().remove("driver_claimed");
15146 runner.state = serde_json::from_value(json).unwrap();
15147 assert_eq!(runner.state.driver_claimed, None);
15148 runner.state.save().unwrap();
15149 let mut execution = runner
15150 .claim_driver()
15151 .await
15152 .expect("the execution lease, not a missing marker, excludes a second driver");
15153 assert_eq!(execution.runner.state.passes.len(), 1);
15154 execution
15155 .driver
15156 .finish(&mut execution.runner.state)
15157 .await
15158 .unwrap();
15159 }
15160
15161 #[tokio::test]
15162 async fn driver_admission_refuses_a_proven_live_driver() {
15163 crate::run::pin_test_home();
15164 let mut runner = runner_at(RunStatus::Reviewing);
15165 let pid = std::process::id();
15166 runner.state.driver_pid = Some(pid);
15167 runner.state.driver_started_at = crate::proc::process_started_at(pid);
15168 assert!(runner.state.driver_started_at.is_some());
15169 runner.state.save().unwrap();
15170 let error = runner
15171 .claim_driver()
15172 .await
15173 .err()
15174 .expect("a live driver must refuse");
15175 assert!(error.to_string().contains("live driver"), "{error:#}");
15176 let saved = RunState::load(&runner.state.id).unwrap();
15177 assert_eq!(saved.driver_pid, Some(pid));
15178 assert_eq!(saved.status, RunStatus::Reviewing);
15179 assert!(saved.passes.is_empty());
15180 }
15181
15182 #[test]
15183 fn parking_resource_halt_persists_the_pass_once() {
15184 crate::run::pin_test_home();
15185 let mut runner = runner_at(RunStatus::Reviewing);
15186 runner.state.begin_pass();
15187 runner.state.resource_block = Some(crate::run::ResourceBlock {
15188 node: "fix".into(),
15189 kind: "shared build cache".into(),
15190 detail: "cancelled".into(),
15191 at: Timestamp::now(),
15192 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15193 });
15194 let cause = ParkCause::Upgrade {
15195 from: "old".into(),
15196 to: Some("new".into()),
15197 };
15198 runner.pause.park_for(cause.clone());
15199 runner.stop_for_resource().unwrap();
15202 assert!(runner.resource_halt().unwrap());
15203 assert!(runner.park_here().unwrap());
15204 let saved = RunState::load(&runner.state.id).unwrap();
15205 assert!(saved.parked);
15206 assert_eq!(saved.status, RunStatus::Reviewing);
15207 assert!(saved.passes[0].ended_at.is_some());
15208 assert_eq!(
15209 saved.passes[0].end,
15210 Some(crate::run::PassEnd::Parked {
15211 cause: cause.clone()
15212 })
15213 );
15214 assert_eq!(
15215 saved.resource_block.unwrap().cause,
15216 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15217 );
15218 assert_eq!(
15219 saved
15220 .events
15221 .iter()
15222 .filter(|event| event.node == "park")
15223 .count(),
15224 1
15225 );
15226 }
15227
15228 #[test]
15236 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15237 crate::run::pin_test_home();
15238 let mut runner = runner_at(RunStatus::Implementing);
15239 let shutdown = Pause::new();
15240 runner.on_pause(shutdown.clone());
15241 let interrupt = Pause::new();
15242 runner.watch_interrupt(interrupt.clone());
15243
15244 assert!(!runner.park_here().expect("park_here"));
15246 assert!(!runner.state.parked);
15247
15248 interrupt.park_because("test");
15250 assert!(!shutdown.parked());
15251 assert!(runner.park_here().expect("park_here"));
15252 }
15253
15254 #[tokio::test]
15268 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15269 crate::run::pin_test_home();
15270 let mut runner = runner_at(RunStatus::Implementing);
15271 let interrupt = Pause::new();
15272 runner.watch_interrupt(interrupt.clone());
15273
15274 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15275 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15276
15277 let node = async move {
15281 started_tx.send(()).expect("send started");
15282 finish_rx.await.expect("recv finish");
15283 "node finished"
15284 };
15285
15286 let interrupter = async move {
15287 started_rx.await.expect("recv started");
15288 interrupt.park_because("higher-priority task waiting");
15290 tokio::task::yield_now().await;
15294 finish_tx.send(()).expect("send finish");
15295 };
15296
15297 let (node_result, ()) = tokio::join!(node, interrupter);
15298 assert_eq!(
15299 node_result, "node finished",
15300 "the in-flight call ran to completion"
15301 );
15302
15303 assert!(runner.park_here().expect("park_here"));
15306 assert!(runner.state.parked);
15307 }
15308
15309 #[test]
15315 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15316 crate::run::pin_test_home();
15317 let mut runner = runner_at(RunStatus::Judging);
15318 runner.state.config.agents = vec![conductor()];
15322 runner.state.candidates = vec![Candidate {
15323 index: 0,
15324 label: 'A',
15325 agent: "alpha".to_owned(),
15326 branch: "magi/x/A".to_owned(),
15327 worktree: PathBuf::from("/nonexistent/worktree"),
15328 summary: "did the thing".to_owned(),
15329 stat: "1 file changed".to_owned(),
15330 files: 1,
15331 commits: 1,
15332 empty: false,
15333 failed: None,
15334 verified_noop: None,
15335 duration_ms: 1234,
15336 folded: false,
15337 }];
15338 let run_id = runner.state.id.clone();
15339
15340 let interrupt = Pause::new();
15341 runner.watch_interrupt(interrupt.clone());
15342 interrupt.park_because("task c3d4 asked to run first");
15343 assert!(runner.park_here().expect("park_here"));
15344
15345 let resumed = Runner::resume(&run_id).expect("resume");
15346 assert_eq!(resumed.state.candidates.len(), 1);
15347 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15348 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15349 assert_eq!(resumed.state.status, runner.state.status);
15350 assert!(
15351 resumed.state.parked,
15352 "still parked until `execute` actually walks the graph again"
15353 );
15354 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15355 }
15356
15357 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15359 let mut q = ask::Question::new(
15360 run.to_owned(),
15361 "implement".to_owned(),
15362 "impl-A".to_owned(),
15363 "Which storage backend should the cache use?".to_owned(),
15364 String::new(),
15365 vec!["SQLite".to_owned(), "Redis".to_owned()],
15366 );
15367 store.put(&mut q).unwrap();
15368 q
15369 }
15370
15371 #[test]
15372 fn a_failed_runs_open_question_is_abandoned() {
15373 ask_test_home();
15374 let store = ask::Questions::open();
15375 let mut runner = runner_at(RunStatus::Failed);
15376 let run = runner.state.id.clone();
15377 let q = ask_open_question(&store, &run);
15378
15379 runner.settle_questions();
15380
15381 let back = store.get(&q.id).unwrap();
15382 assert!(
15383 !back.status.open(),
15384 "the seat that asked died with the run; nobody is left to read an answer"
15385 );
15386 assert!(
15387 back.detail.contains(&run) && back.detail.contains("failed"),
15388 "the reason names what the run became, not just that it is gone: {}",
15389 back.detail
15390 );
15391 }
15392
15393 #[test]
15394 fn a_merged_runs_open_question_is_abandoned_too() {
15395 ask_test_home();
15396 let store = ask::Questions::open();
15397 for status in [RunStatus::Merged, RunStatus::Ready] {
15400 let mut runner = runner_at(status);
15401 let run = runner.state.id.clone();
15402 let q = ask_open_question(&store, &run);
15403
15404 runner.settle_questions();
15405
15406 let back = store.get(&q.id).unwrap();
15407 assert!(
15408 !back.status.open(),
15409 "{status:?} run's question must not outlive the run"
15410 );
15411 }
15412 }
15413
15414 #[test]
15415 fn a_still_resumable_runs_open_question_is_left_alone() {
15416 ask_test_home();
15417 let store = ask::Questions::open();
15418 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15424 let mut runner = runner_at(status);
15425 let run = runner.state.id.clone();
15426 let q = ask_open_question(&store, &run);
15427
15428 runner.settle_questions();
15429
15430 let back = store.get(&q.id).unwrap();
15431 assert!(
15432 back.status.open(),
15433 "{status:?} is still alive; the question must still be waiting"
15434 );
15435 }
15436 }
15437
15438 #[test]
15439 fn settle_questions_never_touches_an_already_answered_question() {
15440 ask_test_home();
15441 let store = ask::Questions::open();
15442 let mut runner = runner_at(RunStatus::Failed);
15443 let run = runner.state.id.clone();
15444 let mut q = ask_open_question(&store, &run);
15445 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15446 .unwrap();
15447 store.put(&mut q).unwrap();
15448
15449 runner.settle_questions();
15454 runner.settle_questions();
15455
15456 let back = store.get(&q.id).unwrap();
15457 assert_eq!(
15458 back.status,
15459 ask::QuestionStatus::Answered,
15460 "a real answer is a decision on record, never overwritten by a sweep"
15461 );
15462 }
15463
15464 #[tokio::test]
15475 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15476 crate::run::pin_test_home();
15477 let tmp = tempfile::tempdir().expect("tempdir");
15478 let repo = tmp.path().join("repo");
15479 std::fs::create_dir_all(&repo).unwrap();
15480 init_repo(&repo);
15481
15482 let mut config = Config::default();
15483 config.graph.worktree_root = Some(tmp.path().join("wt"));
15484
15485 let mut state = RunState::new(
15486 repo.clone(),
15487 "main".to_owned(),
15488 "deadbeef".to_owned(),
15489 "task".to_owned(),
15490 config,
15491 );
15492 let root = state.worktree_root();
15493 let wt_a = root.join("cand-A");
15494 let wt_b = root.join("cand-B");
15495 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15496 .await
15497 .expect("worktree A");
15498 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15499 .await
15500 .expect("worktree B");
15501
15502 state.candidates = vec![
15503 Candidate {
15504 index: 0,
15505 label: 'A',
15506 agent: "alpha".to_owned(),
15507 branch: "magi/x/A".to_owned(),
15508 worktree: wt_a.clone(),
15509 summary: String::new(),
15510 stat: String::new(),
15511 files: 0,
15512 commits: 0,
15513 empty: false,
15514 failed: None,
15515 verified_noop: None,
15516 duration_ms: 0,
15517 folded: false,
15518 },
15519 Candidate {
15520 index: 1,
15521 label: 'B',
15522 agent: "beta".to_owned(),
15523 branch: "magi/x/B".to_owned(),
15524 worktree: wt_b.clone(),
15525 summary: String::new(),
15526 stat: String::new(),
15527 files: 0,
15528 commits: 0,
15529 empty: false,
15530 failed: None,
15531 verified_noop: None,
15532 duration_ms: 0,
15533 folded: false,
15534 },
15535 ];
15536 state.tally = Some(Tally {
15537 first_choice: BTreeMap::from([('A', 1)]),
15538 borda: BTreeMap::new(),
15539 winner: 'A',
15540 rankings: 1,
15541 unanimous_initial: true,
15542 deliberated: false,
15543 changed_votes: 0,
15544 unanimous_final: true,
15545 tie_break: None,
15546 judges: 1,
15547 present: 1,
15548 quorum: 1,
15549 met_quorum: true,
15550 uncontested: None,
15551 });
15552 state.status = RunStatus::Ready;
15553
15554 fold_run(&mut state, false, &crate::run::home())
15555 .await
15556 .expect("fold_run");
15557
15558 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15559 assert!(
15560 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15561 "the unmerged winner's branch survives"
15562 );
15563 assert!(
15564 !state.candidates[0].folded,
15565 "the winner is not marked folded"
15566 );
15567
15568 assert!(!wt_b.exists(), "the loser's worktree is removed");
15569 assert!(
15570 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15571 "the loser's branch is removed"
15572 );
15573 assert!(state.candidates[1].folded, "the loser is marked folded");
15574 }
15575
15576 #[tokio::test]
15577 #[cfg(unix)]
15578 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15579 use std::os::fd::AsRawFd as _;
15580 use std::time::Duration;
15581
15582 let tmp = tempfile::tempdir().unwrap();
15583 let repo = tmp.path().join("repo");
15584 std::fs::create_dir_all(&repo).unwrap();
15585 init_repo(&repo);
15586 let common = repo.join(".git");
15587 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15588 let home = tmp.path().join("home");
15589 let mut config = Config::default();
15590 config.coordination = crate::config::test_coordination();
15591 config.graph.worktree_root = Some(tmp.path().join("wt"));
15592 config.graph.timeout_resource_wait = 1;
15593 let mut state = RunState::new(
15594 repo.clone(),
15595 "main".into(),
15596 "deadbeef".into(),
15597 "task".into(),
15598 config,
15599 );
15600 let root = state.worktree_root();
15601 let candidate = root.join("cand-A");
15602 let reviewer = root.join("review-1");
15603 let branch = state.branch_for('A');
15604 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15605 .await
15606 .unwrap();
15607 git::worktree_add_detached(&repo, &reviewer, "main")
15608 .await
15609 .unwrap();
15610 git::acquire_worktree_config(&repo).await.unwrap();
15611 state.enabled_worktree_config = true;
15612 state.status = RunStatus::Ready;
15613 state.candidates.push(Candidate {
15614 index: 0,
15615 label: 'A',
15616 agent: "implement".into(),
15617 branch: branch.clone(),
15618 worktree: candidate.clone(),
15619 summary: String::new(),
15620 stat: String::new(),
15621 files: 0,
15622 commits: 0,
15623 empty: false,
15624 failed: None,
15625 verified_noop: None,
15626 duration_ms: 0,
15627 folded: false,
15628 });
15629 state.save_under(&home).unwrap();
15630 let config_before = std::fs::read(common.join("config")).unwrap();
15631 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15632 .await
15633 .unwrap();
15634 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15637 assert_eq!(
15638 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15639 0
15640 );
15641 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15642 assert!(
15643 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15644 .await
15645 .is_err()
15646 );
15647 assert!(candidate.exists());
15648 assert!(reviewer.exists());
15649 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15650 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15651 assert_eq!(
15652 git::git(&repo, &["worktree", "list", "--porcelain"])
15653 .await
15654 .unwrap(),
15655 worktrees_before
15656 );
15657 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15658 .await
15659 .unwrap()
15660 .unwrap_err();
15661 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15662 drop(fold);
15663 assert!(candidate.exists() && reviewer.exists());
15664 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15665 assert_eq!(
15666 state.resource_block.as_ref().unwrap().kind,
15667 "repository coordination"
15668 );
15669 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15670 drop(external);
15671 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15672 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15673 .await
15674 .unwrap()
15675 .unwrap();
15676 drop(fold);
15677 assert!(removed.contains(&branch));
15678 assert!(!root.exists());
15679 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15680 assert!(state.candidates[0].folded);
15681 assert!(!state.enabled_worktree_config);
15682 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15683 let enabled = git::git_raw(
15684 &repo,
15685 &["config", "--local", "--get", "extensions.worktreeConfig"],
15686 )
15687 .await
15688 .unwrap();
15689 assert!(!enabled.ok(), "the last config reference was released");
15690 }
15691
15692 #[tokio::test]
15695 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15696 let tmp = tempfile::tempdir().expect("tempdir");
15697 let repo = tmp.path().join("repo");
15698 std::fs::create_dir_all(&repo).unwrap();
15699 init_repo(&repo);
15700 let home = tmp.path().join("home");
15701
15702 let mut config = Config::default();
15703 config.graph.worktree_root = Some(tmp.path().join("wt"));
15704 let mut state = RunState::new(
15705 repo.clone(),
15706 "main".to_owned(),
15707 "deadbeef".to_owned(),
15708 "task".to_owned(),
15709 config,
15710 );
15711 git::git(&repo, &["branch", "magi/x/A", "main"])
15713 .await
15714 .expect("branch");
15715 state.candidates = vec![Candidate {
15716 index: 0,
15717 label: 'A',
15718 agent: "alpha".to_owned(),
15719 branch: "magi/x/A".to_owned(),
15720 worktree: state.worktree_root().join("cand-A"),
15721 summary: String::new(),
15722 stat: String::new(),
15723 files: 0,
15724 commits: 0,
15725 empty: false,
15726 failed: None,
15727 verified_noop: None,
15728 duration_ms: 0,
15729 folded: true,
15730 }];
15731 state.released_to = Some("20260901-000000-new1".to_owned());
15732 state.released_branches = vec!["magi/x/A".to_owned()];
15733
15734 fold_run(&mut state, true, &home).await.expect("fold_run");
15735 state.status = RunStatus::Merged;
15736 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15737 state.merge = Some(crate::run::MergeOutcome {
15738 mode: MergeMode::Pr,
15739 ok: true,
15740 detail: "confirmed merged".into(),
15741 empty: false,
15742 });
15743 fold_run(&mut state, true, &home)
15744 .await
15745 .expect("merged fold");
15746
15747 assert!(
15748 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15749 "the handed-over branch survives a fold"
15750 );
15751 }
15752
15753 #[tokio::test]
15757 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15758 let tmp = tempfile::tempdir().expect("tempdir");
15759 let repo = tmp.path().join("repo");
15760 std::fs::create_dir_all(&repo).unwrap();
15761 init_repo(&repo);
15762 let run = |args: &[&str]| {
15763 let out = std::process::Command::new("git")
15764 .quiet()
15765 .args(args)
15766 .current_dir(&repo)
15767 .output()
15768 .expect("spawn git");
15769 assert!(out.status.success(), "git {args:?}");
15770 };
15771 run(&["branch", "magi/x/A"]);
15772 run(&["checkout", "-q", "-b", "magi/x/B"]);
15773 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15774 run(&["add", "-A"]);
15775 run(&["commit", "-q", "-m", "work"]);
15776 run(&["checkout", "-q", "main"]);
15777
15778 let probe = RunState::new(
15782 repo.clone(),
15783 "main".to_owned(),
15784 "deadbeef".to_owned(),
15785 "task".to_owned(),
15786 Config::default(),
15787 );
15788 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15789 let bare = tmp.path().join("origin.git");
15790 let out = std::process::Command::new("git")
15791 .quiet()
15792 .args(["init", "-q", "--bare"])
15793 .arg(&bare)
15794 .output()
15795 .expect("spawn git");
15796 assert!(out.status.success(), "git init --bare");
15797 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15798 run(&["push", "-q", "origin", "main"]);
15799
15800 let mut state = RunState::new(
15801 repo.clone(),
15802 "main".to_owned(),
15803 "deadbeef".to_owned(),
15804 "task".to_owned(),
15805 Config::default(),
15806 );
15807 state.seeds = vec![refs::Seed {
15808 token: "magi/27b2/A".to_owned(),
15809 kind: refs::SeedKind::Unresolved,
15810 sha: String::new(),
15811 branch: true,
15812 detail: "no branch or commit named magi/27b2/A".to_owned(),
15813 }];
15814
15815 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15816 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15817 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15818 let detail = empty_candidate_detail(&state, "main");
15819 assert!(detail.starts_with("empty candidate"), "{detail}");
15820 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15821 assert!(detail.contains("--start-from"), "{detail}");
15822 }
15823
15824 #[tokio::test]
15833 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15834 let tmp = tempfile::tempdir().expect("tempdir");
15835 let repo = tmp.path().join("repo");
15836 std::fs::create_dir_all(&repo).unwrap();
15837 init_repo(&repo);
15838
15839 let mut config = Config::default();
15840 config.merge.mode = MergeMode::Local;
15841
15842 let mut state = RunState::new(
15843 repo.clone(),
15844 "main".to_owned(),
15845 "deadbeef".to_owned(),
15846 "task".to_owned(),
15847 config,
15848 );
15849 state.candidates = vec![Candidate {
15850 index: 0,
15851 label: 'A',
15852 agent: "alpha".to_owned(),
15853 branch: "does-not-exist".to_owned(),
15854 worktree: repo.clone(),
15855 summary: String::new(),
15856 stat: String::new(),
15857 files: 0,
15858 commits: 0,
15859 empty: false,
15860 failed: None,
15861 verified_noop: None,
15862 duration_ms: 0,
15863 folded: false,
15864 }];
15865 state.tally = Some(Tally {
15866 first_choice: BTreeMap::from([('A', 1)]),
15867 borda: BTreeMap::new(),
15868 winner: 'A',
15869 rankings: 1,
15870 unanimous_initial: true,
15871 deliberated: false,
15872 changed_votes: 0,
15873 unanimous_final: true,
15874 tie_break: None,
15875 judges: 0,
15876 present: 0,
15877 quorum: 0,
15878 met_quorum: true,
15879 uncontested: Some("only candidate A produced a change".to_owned()),
15880 });
15881 state.reviews = vec![ReviewRound {
15882 round: 1,
15883 head: "deadbeef".to_owned(),
15884 verified_head: None,
15885 verified_at: None,
15886 reviews: Vec::new(),
15887 e2e: Vec::new(),
15888 fix: None,
15889 blocking: 0,
15890 answered: 0,
15891 expected: 0,
15892 clean: true,
15893 verify_retried: false,
15894 e2e_deferred: false,
15895 e2e_defer_reason: None,
15896 progressed: false,
15897 vote_split: false,
15898 reconsideration: Vec::new(),
15899 verdict: None,
15900 }];
15901 state.gate = vec![CommandOutcome {
15902 command: "test".to_owned(),
15903 code: Some(0),
15904 output_tail: String::new(),
15905 duration_ms: 0,
15906 resource_blocked: false,
15907 }];
15908 state.gate_ran = true;
15909 state.status = RunStatus::Ready;
15914 state.merge = Some(MergeOutcome {
15915 mode: MergeMode::Local,
15916 ok: false,
15917 detail: "already concluded".to_owned(),
15918 empty: false,
15919 });
15920
15921 let mut runner = Runner {
15922 state,
15923 roles: ResolvedRoles {
15924 implementers: Vec::new(),
15925 judges: Vec::new(),
15926 reviewers: Vec::new(),
15927 fixer: None,
15928 conductor: conductor(),
15929 implementer_roster: Vec::new(),
15930 judge_roster: Vec::new(),
15931 reviewer_roster: Vec::new(),
15932 },
15933 sem: Arc::new(Semaphore::new(1)),
15934 pause: Pause::new(),
15935 interrupt: Pause::new(),
15936 };
15937
15938 runner.merge().await.expect("merge");
15939
15940 assert_eq!(
15941 runner.state.status,
15942 RunStatus::Ready,
15943 "a concluded run's status must not change on reentry"
15944 );
15945 assert_eq!(
15946 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15947 Some("already concluded"),
15948 "merge must not run again once the node already recorded an outcome"
15949 );
15950 }
15951
15952 #[tokio::test]
15961 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15962 let tmp = tempfile::tempdir().expect("tempdir");
15963 let repo = tmp.path().join("repo");
15964 std::fs::create_dir_all(&repo).unwrap();
15965 init_repo(&repo);
15966
15967 let mut config = Config::default();
15968 config.merge.mode = MergeMode::Local;
15969
15970 let mut state = RunState::new(
15971 repo.clone(),
15972 "main".to_owned(),
15973 "deadbeef".to_owned(),
15974 "task".to_owned(),
15975 config,
15976 );
15977 state.candidates = vec![Candidate {
15978 index: 0,
15979 label: 'A',
15980 agent: "alpha".to_owned(),
15981 branch: "does-not-exist".to_owned(),
15982 worktree: repo.clone(),
15983 summary: String::new(),
15984 stat: String::new(),
15985 files: 0,
15986 commits: 0,
15987 empty: false,
15988 failed: None,
15989 verified_noop: None,
15990 duration_ms: 0,
15991 folded: false,
15992 }];
15993 state.tally = Some(Tally {
15994 first_choice: BTreeMap::from([('A', 1)]),
15995 borda: BTreeMap::new(),
15996 winner: 'A',
15997 rankings: 1,
15998 unanimous_initial: true,
15999 deliberated: false,
16000 changed_votes: 0,
16001 unanimous_final: true,
16002 tie_break: None,
16003 judges: 0,
16004 present: 0,
16005 quorum: 0,
16006 met_quorum: true,
16007 uncontested: Some("only candidate A produced a change".to_owned()),
16008 });
16009 state.reviews = vec![ReviewRound {
16010 round: 1,
16011 head: "deadbeef".to_owned(),
16012 verified_head: None,
16013 verified_at: None,
16014 reviews: Vec::new(),
16015 e2e: Vec::new(),
16016 fix: None,
16017 blocking: 0,
16018 answered: 0,
16019 expected: 0,
16020 clean: true,
16021 verify_retried: false,
16022 e2e_deferred: false,
16023 e2e_defer_reason: None,
16024 progressed: false,
16025 vote_split: false,
16026 reconsideration: Vec::new(),
16027 verdict: None,
16028 }];
16029 state.gate = Vec::new();
16031 state.gate_ran = false;
16032 state.status = RunStatus::Gating;
16033
16034 let mut runner = Runner {
16035 state,
16036 roles: ResolvedRoles {
16037 implementers: Vec::new(),
16038 judges: Vec::new(),
16039 reviewers: Vec::new(),
16040 fixer: None,
16041 conductor: conductor(),
16042 implementer_roster: Vec::new(),
16043 judge_roster: Vec::new(),
16044 reviewer_roster: Vec::new(),
16045 },
16046 sem: Arc::new(Semaphore::new(1)),
16047 pause: Pause::new(),
16048 interrupt: Pause::new(),
16049 };
16050
16051 runner.merge().await.expect("merge");
16052
16053 assert!(
16054 runner.state.merge.is_none(),
16055 "an empty gate must never be read as a passing one: {:?}",
16056 runner.state.merge
16057 );
16058 }
16059
16060 #[tokio::test]
16067 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16068 let tmp = tempfile::tempdir().expect("tempdir");
16069 let repo = tmp.path().join("repo");
16070 std::fs::create_dir_all(&repo).unwrap();
16071 init_repo(&repo);
16072
16073 let config = Config::default();
16075
16076 let mut state = RunState::new(
16077 repo.clone(),
16078 "main".to_owned(),
16079 "deadbeef".to_owned(),
16080 "task".to_owned(),
16081 config,
16082 );
16083 state.candidates = vec![Candidate {
16084 index: 0,
16085 label: 'A',
16086 agent: "alpha".to_owned(),
16087 branch: "does-not-exist".to_owned(),
16088 worktree: repo.clone(),
16089 summary: String::new(),
16090 stat: String::new(),
16091 files: 0,
16092 commits: 0,
16093 empty: false,
16094 failed: None,
16095 verified_noop: None,
16096 duration_ms: 0,
16097 folded: false,
16098 }];
16099 state.tally = Some(Tally {
16100 first_choice: BTreeMap::from([('A', 1)]),
16101 borda: BTreeMap::new(),
16102 winner: 'A',
16103 rankings: 1,
16104 unanimous_initial: true,
16105 deliberated: false,
16106 changed_votes: 0,
16107 unanimous_final: true,
16108 tie_break: None,
16109 judges: 0,
16110 present: 0,
16111 quorum: 0,
16112 met_quorum: true,
16113 uncontested: Some("only candidate A produced a change".to_owned()),
16114 });
16115 state.reviews = vec![ReviewRound {
16116 round: 1,
16117 head: "deadbeef".to_owned(),
16118 verified_head: None,
16119 verified_at: None,
16120 reviews: Vec::new(),
16121 e2e: Vec::new(),
16122 fix: None,
16123 blocking: 0,
16124 answered: 0,
16125 expected: 0,
16126 clean: true,
16127 verify_retried: false,
16128 e2e_deferred: false,
16129 e2e_defer_reason: None,
16130 progressed: false,
16131 vote_split: false,
16132 reconsideration: Vec::new(),
16133 verdict: None,
16134 }];
16135
16136 let mut runner = Runner {
16137 state,
16138 roles: ResolvedRoles {
16139 implementers: Vec::new(),
16140 judges: Vec::new(),
16141 reviewers: Vec::new(),
16142 fixer: None,
16143 conductor: conductor(),
16144 implementer_roster: Vec::new(),
16145 judge_roster: Vec::new(),
16146 reviewer_roster: Vec::new(),
16147 },
16148 sem: Arc::new(Semaphore::new(1)),
16149 pause: Pause::new(),
16150 interrupt: Pause::new(),
16151 };
16152
16153 runner.gate().await.expect("gate");
16154 assert!(
16155 runner.state.gate_ran,
16156 "zero configured commands is still a real attempt, not an unrun gate"
16157 );
16158 assert!(runner.state.gate.is_empty());
16159 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16160 assert_ne!(
16161 runner.state.status,
16162 RunStatus::Blocked,
16163 "a gate with nothing to check must not read as failed"
16164 );
16165
16166 runner.merge().await.expect("merge");
16167 assert_eq!(
16168 runner.state.status,
16169 RunStatus::Ready,
16170 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16171 );
16172 }
16173
16174 #[tokio::test]
16185 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16186 crate::run::pin_test_home();
16187 let home = crate::run::home();
16188
16189 let tmp = tempfile::tempdir().expect("tempdir");
16190 let repo = tmp.path().join("repo");
16191 std::fs::create_dir_all(&repo).unwrap();
16192 init_repo(&repo);
16193 let cache_dir = tmp.path().join("target");
16196
16197 let mut config = Config::default();
16198 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16199 config.graph.timeout_verify = Some(2);
16202 config.graph.timeout_resource_wait = 2;
16203
16204 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16205 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16206 .expect("no io error acquiring directly")
16207 {
16208 crate::cache::AcquireOutcome::Acquired(g) => g,
16209 crate::cache::AcquireOutcome::Busy(b) => {
16210 panic!("expected the direct acquire to win the lease first: {b:?}")
16211 }
16212 };
16213
16214 let mut state = RunState::new(
16215 repo.clone(),
16216 "main".to_owned(),
16217 "deadbeef".to_owned(),
16218 "task".to_owned(),
16219 config,
16220 );
16221 state.candidates = vec![Candidate {
16222 index: 0,
16223 label: 'A',
16224 agent: "alpha".to_owned(),
16225 branch: "does-not-exist".to_owned(),
16226 worktree: repo.clone(),
16227 summary: String::new(),
16228 stat: String::new(),
16229 files: 0,
16230 commits: 0,
16231 empty: false,
16232 failed: None,
16233 verified_noop: None,
16234 duration_ms: 0,
16235 folded: false,
16236 }];
16237 state.tally = Some(Tally {
16238 first_choice: BTreeMap::from([('A', 1)]),
16239 borda: BTreeMap::new(),
16240 winner: 'A',
16241 rankings: 1,
16242 unanimous_initial: true,
16243 deliberated: false,
16244 changed_votes: 0,
16245 unanimous_final: true,
16246 tie_break: None,
16247 judges: 0,
16248 present: 0,
16249 quorum: 0,
16250 met_quorum: true,
16251 uncontested: Some("only candidate A produced a change".to_owned()),
16252 });
16253 state.reviews = vec![ReviewRound {
16254 round: 1,
16255 head: "deadbeef".to_owned(),
16256 verified_head: None,
16257 verified_at: None,
16258 reviews: Vec::new(),
16259 e2e: Vec::new(),
16260 fix: None,
16261 blocking: 0,
16262 answered: 0,
16263 expected: 0,
16264 clean: true,
16265 verify_retried: false,
16266 e2e_deferred: false,
16267 e2e_defer_reason: None,
16268 progressed: false,
16269 vote_split: false,
16270 reconsideration: Vec::new(),
16271 verdict: None,
16272 }];
16273
16274 let mut runner = Runner {
16275 state,
16276 roles: ResolvedRoles {
16277 implementers: Vec::new(),
16278 judges: Vec::new(),
16279 reviewers: Vec::new(),
16280 fixer: None,
16281 conductor: conductor(),
16282 implementer_roster: Vec::new(),
16283 judge_roster: Vec::new(),
16284 reviewer_roster: Vec::new(),
16285 },
16286 sem: Arc::new(Semaphore::new(1)),
16287 pause: Pause::new(),
16288 interrupt: Pause::new(),
16289 };
16290
16291 let started = std::time::Instant::now();
16292 runner.gate().await.expect("gate");
16293 assert!(
16294 started.elapsed() < Duration::from_secs(1),
16295 "a gate with nothing to run must never wait on a lease it never needed"
16296 );
16297 assert!(
16298 runner.state.gate_ran,
16299 "zero commands is still a real, immediate attempt"
16300 );
16301 assert!(runner.state.gate.is_empty());
16302 assert_ne!(
16303 runner.state.status,
16304 RunStatus::Blocked,
16305 "must not read as resource-blocked on a lease it never asked for"
16306 );
16307 }
16308
16309 #[cfg(unix)]
16310 #[tokio::test]
16311 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16312 crate::run::pin_test_home();
16313 let tmp = tempfile::tempdir().unwrap();
16314 let mut config = Config::default();
16315 config.disk.min_free_bytes = 0;
16316 let mut state = RunState::new(
16317 tmp.path().to_owned(),
16318 "main".into(),
16319 "base".into(),
16320 "verification evidence".into(),
16321 config,
16322 );
16323 let (outcomes, pids) = run_commands(
16324 &mut state,
16325 "gate",
16326 "gate",
16327 1,
16328 &["sh".into(), "-c".into()],
16329 &[
16330 "printf before-timeout; exec sleep 30".into(),
16331 "kill -TERM $$".into(),
16332 "exit 7".into(),
16333 "printf success".into(),
16334 ],
16335 tmp.path(),
16336 Duration::from_secs(1),
16337 )
16338 .await;
16339 wait_for_timed_out_children_to_die(&pids).await;
16340 assert_eq!(
16341 pids.len(),
16342 1,
16343 "unleased timeout retains its death-check PID"
16344 );
16345 assert_eq!(outcomes.len(), 4);
16346 assert_eq!(outcomes[0].code, None);
16347 assert!(outcomes[0].output_tail.contains("before-timeout"));
16348 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16349 assert_eq!(outcomes[1].code, None);
16350 assert!(outcomes[1].output_tail.contains("command terminated:"));
16351 assert!(!outcomes[1].output_tail.contains("timed out"));
16352 assert_eq!(outcomes[2].code, Some(7));
16353 assert!(!outcomes[2].output_tail.contains("timed out"));
16354 assert_eq!(outcomes[3].code, Some(0));
16355 assert_eq!(outcomes[3].output_tail, "success");
16356 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16357 }
16358
16359 #[tokio::test]
16369 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16370 crate::run::pin_test_home();
16371
16372 let tmp = tempfile::tempdir().expect("tempdir");
16373 let repo = tmp.path().join("repo");
16374 std::fs::create_dir_all(&repo).unwrap();
16375 init_repo(&repo);
16376
16377 let mut config = Config::default();
16378 config.verify.gate = vec![
16379 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16380 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16381 .to_owned(),
16382 ];
16383
16384 let mut state = RunState::new(
16385 repo.clone(),
16386 "main".to_owned(),
16387 "deadbeef".to_owned(),
16388 "task".to_owned(),
16389 config,
16390 );
16391 let run_id = state.id.clone();
16392 state.candidates = vec![Candidate {
16393 index: 0,
16394 label: 'A',
16395 agent: "alpha".to_owned(),
16396 branch: "does-not-exist".to_owned(),
16397 worktree: repo.clone(),
16398 summary: String::new(),
16399 stat: String::new(),
16400 files: 0,
16401 commits: 0,
16402 empty: false,
16403 failed: None,
16404 verified_noop: None,
16405 duration_ms: 0,
16406 folded: false,
16407 }];
16408 state.tally = Some(Tally {
16409 first_choice: BTreeMap::from([('A', 1)]),
16410 borda: BTreeMap::new(),
16411 winner: 'A',
16412 rankings: 1,
16413 unanimous_initial: true,
16414 deliberated: false,
16415 changed_votes: 0,
16416 unanimous_final: true,
16417 tie_break: None,
16418 judges: 0,
16419 present: 0,
16420 quorum: 0,
16421 met_quorum: true,
16422 uncontested: Some("only candidate A produced a change".to_owned()),
16423 });
16424 state.reviews = vec![ReviewRound {
16425 round: 1,
16426 head: "deadbeef".to_owned(),
16427 verified_head: None,
16428 verified_at: None,
16429 reviews: Vec::new(),
16430 e2e: Vec::new(),
16431 fix: None,
16432 blocking: 0,
16433 answered: 0,
16434 expected: 0,
16435 clean: true,
16436 verify_retried: false,
16437 e2e_deferred: false,
16438 e2e_defer_reason: None,
16439 progressed: false,
16440 vote_split: false,
16441 reconsideration: Vec::new(),
16442 verdict: None,
16443 }];
16444
16445 let mut runner = Runner {
16446 state,
16447 roles: ResolvedRoles {
16448 implementers: Vec::new(),
16449 judges: Vec::new(),
16450 reviewers: Vec::new(),
16451 fixer: None,
16452 conductor: conductor(),
16453 implementer_roster: Vec::new(),
16454 judge_roster: Vec::new(),
16455 reviewer_roster: Vec::new(),
16456 },
16457 sem: Arc::new(Semaphore::new(1)),
16458 pause: Pause::new(),
16459 interrupt: Pause::new(),
16460 };
16461
16462 let started_marker = repo.join("started.marker");
16463 let release_marker = repo.join("release.marker");
16464 let poller = tokio::spawn(async move {
16465 for _ in 0..100 {
16470 if started_marker.exists()
16471 && let Ok(s) = crate::run::RunState::load(&run_id)
16472 && let Some(a) = s.active.get("gate")
16473 {
16474 std::fs::write(&release_marker, b"go").expect("release marker");
16475 return Some(a.clone());
16476 }
16477 tokio::time::sleep(Duration::from_millis(50)).await;
16478 }
16479 None
16480 });
16481
16482 runner.gate().await.expect("gate");
16483 let captured = poller.await.expect("poller task");
16484 let captured = captured.expect(
16485 "the poller never saw a `gate` task entry in run.json while the command was \
16486 still blocked on its own release marker",
16487 );
16488
16489 assert_eq!(captured.task.as_deref(), Some("gate"));
16490 assert_eq!(captured.node, "gate");
16491 assert_eq!(captured.index, Some(1));
16492 assert_eq!(captured.total, Some(1));
16493 assert!(
16494 captured
16495 .command
16496 .as_deref()
16497 .is_some_and(|c| c.contains("started.marker")),
16498 "{captured:?}"
16499 );
16500
16501 assert!(
16502 runner.state.active.is_empty(),
16503 "the entry must be cleared once the command actually finished: {:?}",
16504 runner.state.active
16505 );
16506 assert!(runner.state.gate_ran);
16507 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16508 }
16509
16510 #[tokio::test]
16513 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16514 use crate::verdict::{Finding, ReviewVote, Severity};
16515 crate::run::pin_test_home();
16516 let tmp = tempfile::tempdir().expect("tempdir");
16517 let repo = tmp.path().join("repo");
16518 std::fs::create_dir_all(&repo).unwrap();
16519 init_repo(&repo);
16520
16521 for (severity, vote, expect) in [
16522 (Severity::Major, ReviewVote::Reject, true),
16523 (Severity::Minor, ReviewVote::Reject, false),
16524 (Severity::Major, ReviewVote::Approve, false),
16525 ] {
16526 let mut round = review_round(false, 1, 1, 1, false, true);
16527 round.reviews = vec![ReviewRecord {
16528 reviewer: 1,
16529 agent: "alpha".to_owned(),
16530 summary: String::new(),
16531 findings: vec![Finding {
16532 id: "R1-1-1".to_owned(),
16533 severity,
16534 file: None,
16535 line: None,
16536 title: "t".to_owned(),
16537 detail: String::new(),
16538 }],
16539 vote: Some(vote),
16540 failed: None,
16541 duration_ms: 0,
16542 attempts: 0,
16543 }];
16544 let mut state = RunState::new(
16545 repo.clone(),
16546 "main".to_owned(),
16547 "deadbeef".to_owned(),
16548 "task".to_owned(),
16549 Config::default(),
16550 );
16551 state.reviews = vec![round];
16552 let mut runner = Runner {
16553 state,
16554 roles: ResolvedRoles {
16555 implementers: Vec::new(),
16556 judges: Vec::new(),
16557 reviewers: Vec::new(),
16558 fixer: None,
16559 conductor: conductor(),
16560 implementer_roster: Vec::new(),
16561 judge_roster: Vec::new(),
16562 reviewer_roster: Vec::new(),
16563 },
16564 sem: Arc::new(Semaphore::new(1)),
16565 pause: Pause::new(),
16566 interrupt: Pause::new(),
16567 };
16568 let shell = runner.state.config.shell();
16569 runner
16570 .stop_reviewing("round budget spent", &shell, &repo)
16571 .await
16572 .expect("stop_reviewing");
16573 assert_eq!(runner.state.status, RunStatus::Gating);
16574 assert_eq!(
16575 runner.state.contested_handoff.is_some(),
16576 expect,
16577 "{severity:?} + {vote:?}"
16578 );
16579 }
16580 }
16581
16582 #[tokio::test]
16595 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16596 crate::run::pin_test_home();
16597 let home = crate::run::home();
16598
16599 let tmp = tempfile::tempdir().expect("tempdir");
16600 let repo = tmp.path().join("repo");
16601 std::fs::create_dir_all(&repo).unwrap();
16602 init_repo(&repo);
16603 let head = crate::git::rev_parse(&repo, "HEAD")
16604 .await
16605 .expect("rev-parse");
16606 let cache_dir = tmp.path().join("target");
16609
16610 let mut config = Config::default();
16611 config.verify.e2e = vec![format!(
16612 "CARGO_TARGET_DIR='{}' test -f README.md",
16613 cache_dir.display()
16614 )];
16615 config.graph.review_rounds = 1;
16616 config.graph.timeout_verify = Some(2);
16619 config.graph.timeout_resource_wait = 2;
16620
16621 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16622 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16623 .expect("no io error acquiring directly")
16624 {
16625 crate::cache::AcquireOutcome::Acquired(g) => g,
16626 crate::cache::AcquireOutcome::Busy(b) => {
16627 panic!("expected the direct acquire to win the lease first: {b:?}")
16628 }
16629 };
16630
16631 let mut state = RunState::new(
16632 repo.clone(),
16633 "main".to_owned(),
16634 head.clone(),
16635 "task".to_owned(),
16636 config,
16637 );
16638 state.candidates = vec![Candidate {
16639 index: 0,
16640 label: 'A',
16641 agent: "alpha".to_owned(),
16642 branch: "does-not-exist".to_owned(),
16643 worktree: repo.clone(),
16644 summary: String::new(),
16645 stat: String::new(),
16646 files: 0,
16647 commits: 0,
16648 empty: false,
16649 failed: None,
16650 verified_noop: None,
16651 duration_ms: 0,
16652 folded: false,
16653 }];
16654 state.tally = Some(Tally {
16655 first_choice: BTreeMap::from([('A', 1)]),
16656 borda: BTreeMap::new(),
16657 winner: 'A',
16658 rankings: 1,
16659 unanimous_initial: true,
16660 deliberated: false,
16661 changed_votes: 0,
16662 unanimous_final: true,
16663 tie_break: None,
16664 judges: 0,
16665 present: 0,
16666 quorum: 0,
16667 met_quorum: true,
16668 uncontested: Some("only candidate A produced a change".to_owned()),
16669 });
16670 state.reviews = vec![ReviewRound {
16674 round: 1,
16675 head: head.clone(),
16676 verified_head: None,
16677 verified_at: None,
16678 reviews: Vec::new(),
16679 e2e: Vec::new(),
16680 fix: None,
16681 blocking: 1,
16682 answered: 1,
16683 expected: 1,
16684 clean: false,
16685 verify_retried: false,
16686 e2e_deferred: true,
16687 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16688 progressed: false,
16689 vote_split: false,
16690 reconsideration: Vec::new(),
16691 verdict: None,
16692 }];
16693
16694 let mut runner = Runner {
16695 state,
16696 roles: ResolvedRoles {
16697 implementers: Vec::new(),
16698 judges: Vec::new(),
16699 reviewers: Vec::new(),
16700 fixer: None,
16701 conductor: conductor(),
16702 implementer_roster: Vec::new(),
16703 judge_roster: Vec::new(),
16704 reviewer_roster: Vec::new(),
16705 },
16706 sem: Arc::new(Semaphore::new(1)),
16707 pause: Pause::new(),
16708 interrupt: Pause::new(),
16709 };
16710
16711 let shell = runner.state.config.shell();
16712 runner
16713 .stop_reviewing("round budget spent", &shell, &repo)
16714 .await
16715 .expect("stop_reviewing");
16716
16717 let last = runner.state.reviews.last().expect("round record");
16718 assert_eq!(
16719 last.e2e_status(),
16720 E2eStatus::ResourceBlocked,
16721 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16722 failed: {last:?}"
16723 );
16724 assert_eq!(
16725 last.verified_head.as_deref(),
16726 Some(head.as_str()),
16727 "which commit this attempt targeted is known even though nothing finished checking \
16728 it"
16729 );
16730 let first_attempt_at = last
16731 .verified_at
16732 .expect("when this attempt ran is known too");
16733 assert_ne!(
16734 runner.state.status,
16735 RunStatus::Blocked,
16736 "contention is evidence about the machine, not the patch — it must not settle the \
16737 run as blocked: {:?}",
16738 runner.state.status
16739 );
16740 assert!(
16741 !runner
16742 .state
16743 .events
16744 .iter()
16745 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16746 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16747 runner.state.events
16748 );
16749
16750 runner
16755 .stop_reviewing("round budget spent", &shell, &repo)
16756 .await
16757 .expect("stop_reviewing retry");
16758 assert_eq!(
16759 runner.state.reviews.len(),
16760 1,
16761 "no new round was started: {:?}",
16762 runner.state.reviews
16763 );
16764 let last = runner.state.reviews.last().expect("round record");
16765 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16766 assert!(
16767 last.verified_at.expect("still known") > first_attempt_at,
16768 "a second reentry must be a fresh attempt, not a stale copy of the first"
16769 );
16770 assert_ne!(runner.state.status, RunStatus::Blocked);
16771
16772 held.release();
16773 }
16774
16775 #[cfg(unix)]
16776 #[tokio::test]
16777 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
16778 ask_test_home();
16779 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
16780 let tmp = tempfile::tempdir().unwrap();
16781 let repo = tmp.path().join("repo");
16782 std::fs::create_dir(&repo).unwrap();
16783 init_repo(&repo);
16784 let run_git = |args: &[&str]| {
16785 let out = std::process::Command::new("git")
16786 .args(args)
16787 .current_dir(&repo)
16788 .output()
16789 .unwrap();
16790 assert!(
16791 out.status.success(),
16792 "{args:?}: {}",
16793 String::from_utf8_lossy(&out.stderr)
16794 );
16795 };
16796 let origin = tmp.path().join("origin.git");
16797 run_git(&["init", "--bare", origin.to_str().unwrap()]);
16798 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
16799 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
16800 run_git(&["add", "spec.rs"]);
16801 run_git(&["commit", "-m", "initial spec"]);
16802 run_git(&["push", "origin", "main"]);
16803 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16804 run_git(&["checkout", "-b", "fix/review"]);
16805 std::fs::write(
16806 repo.join("check.rs"),
16807 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
16808 )
16809 .unwrap();
16810 run_git(&["add", "check.rs"]);
16811 run_git(&["commit", "-m", "use spec"]);
16812 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
16813 run_git(&["checkout", "main"]);
16814 std::fs::write(
16815 repo.join("spec.rs"),
16816 "struct AgentSpec { max_turn_secs: u64 }\n",
16817 )
16818 .unwrap();
16819 run_git(&["add", "spec.rs"]);
16820 run_git(&["commit", "-m", "extend spec"]);
16821 run_git(&["push", "origin", "main"]);
16822 run_git(&["checkout", "fix/review"]);
16823
16824 let mut runner = runner_at(RunStatus::Reviewing);
16825 runner.state.repo = repo.clone();
16826 runner.state.base_commit = base;
16827 runner.state.pr_head = Some("fix/review".into());
16828 runner.state.config.graph.review_rounds = max_rounds;
16829 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16830 runner.state.config.graph.sessions = false;
16831 runner.state.config.disk.min_free_bytes = 0;
16832 runner.state.config.verify.e2e =
16833 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
16834 runner.state.candidates = vec![Candidate {
16835 index: 0,
16836 label: 'A',
16837 agent: "fixture".into(),
16838 branch: "fix/review".into(),
16839 worktree: repo.clone(),
16840 summary: String::new(),
16841 stat: String::new(),
16842 files: 1,
16843 commits: 1,
16844 empty: false,
16845 failed: None,
16846 verified_noop: None,
16847 duration_ms: 0,
16848 folded: false,
16849 }];
16850 runner.state.tally = Some(Tally {
16851 first_choice: BTreeMap::new(),
16852 borda: BTreeMap::new(),
16853 winner: 'A',
16854 rankings: 0,
16855 unanimous_initial: true,
16856 deliberated: false,
16857 changed_votes: 0,
16858 unanimous_final: true,
16859 tie_break: None,
16860 judges: 0,
16861 present: 0,
16862 quorum: 0,
16863 met_quorum: true,
16864 uncontested: Some("review-only".into()),
16865 });
16866 let mut last = review_round(false, 0, 1, 1, false, false);
16867 last.head = reviewed.clone();
16868 last.e2e_deferred = true;
16869 last.e2e[0].resource_blocked = true;
16870 last.e2e[0].output_tail = "waiting for the shared build cache".into();
16871 if max_rounds == 1 {
16872 last.e2e.clear();
16873 }
16874 runner.state.reviews = vec![last];
16875 let calls = tmp.path().join("fix-calls");
16876 let mut fixer = spec("repair");
16877 fixer
16878 .env
16879 .insert("FIX_CALLS".into(), calls.display().to_string());
16880 fixer.command = vec!["sh".into(), "-c".into(), r#"
16881 echo fix >> "$FIX_CALLS"
16882 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
16883 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
16884 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
16885 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
16886 "#.into()];
16887 let mut reviewer = spec("reviewer");
16888 reviewer.command = vec![
16889 "sh".into(),
16890 "-c".into(),
16891 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
16892 .into(),
16893 ];
16894 runner.roles.fixer = Some(vec![fixer]);
16895 runner.roles.reviewers = vec![reviewer];
16896
16897 runner.sync_to_base().await.unwrap();
16898 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
16899 assert_ne!(rebased, reviewed);
16900 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
16901 if reentry {
16902 let shell = runner.state.config.shell();
16905 runner
16906 .stop_unresolved_review("deferred verification", &shell, &repo)
16907 .await
16908 .unwrap();
16909 assert!(runner.catchup_repairable());
16910 assert!(!calls.exists());
16911 runner.state = RunState::load(&runner.state.id).unwrap();
16912 assert!(runner.catchup_repairable());
16913 }
16914 runner.review_loop().await.unwrap();
16915 let first = &runner.state.reviews[0];
16916 assert_eq!(first.head, reviewed);
16917 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
16918 assert!(
16919 first.verify_retried,
16920 "a reproduced build failure gets exactly one retry: {:?}",
16921 first.e2e
16922 );
16923 assert!(first.e2e[0].output_tail.contains("E0063"));
16924 assert!(!runner.catchup_repairable());
16925 if max_rounds == 2 {
16926 assert_eq!(
16927 runner.state.status,
16928 RunStatus::Gating,
16929 "{}",
16930 crate::report::run(&runner.state)
16931 );
16932 assert_eq!(runner.state.reviews.len(), 2);
16933 assert!(first.fix.as_ref().unwrap().committed);
16934 assert!(runner.state.reviews[1].clean);
16935 assert_eq!(
16936 runner
16937 .state
16938 .reviews
16939 .iter()
16940 .filter(|r| r.fix.is_some())
16941 .count(),
16942 1
16943 );
16944 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
16945 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
16946 } else {
16947 assert_eq!(runner.state.status, RunStatus::Blocked);
16948 assert!(first.fix.is_none());
16949 assert!(!calls.exists());
16950 let recovery = crate::recovery::assess(
16951 "prior",
16952 Some(&runner.state),
16953 &runner.landing_base(),
16954 crate::recovery::Relation::Diverged,
16955 );
16956 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
16957 let reasons = recovery.reasons.join("\n");
16958 assert!(
16959 reasons.contains(&reviewed) && reasons.contains(&rebased),
16960 "{reasons}"
16961 );
16962 assert!(
16963 reasons.contains("base sync to")
16964 && reasons.contains("additional review is required"),
16965 "{reasons}"
16966 );
16967 }
16968 }
16969 }
16970
16971 #[tokio::test]
16983 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16984 crate::run::pin_test_home();
16985 let home = crate::run::home();
16986
16987 let tmp = tempfile::tempdir().expect("tempdir");
16988 let repo = tmp.path().join("repo");
16989 std::fs::create_dir_all(&repo).unwrap();
16990 init_repo(&repo);
16991 let head = crate::git::rev_parse(&repo, "HEAD")
16992 .await
16993 .expect("rev-parse");
16994 let cache_dir = tmp.path().join("target");
16995
16996 let mut config = Config::default();
16997 config.verify.e2e = vec![format!(
16998 "CARGO_TARGET_DIR='{}' test -f README.md",
16999 cache_dir.display()
17000 )];
17001 config.graph.review_rounds = 1;
17002 config.graph.timeout_verify = Some(2);
17003 config.graph.timeout_resource_wait = 2;
17004
17005 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17006 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17007 .expect("no io error acquiring directly")
17008 {
17009 crate::cache::AcquireOutcome::Acquired(g) => g,
17010 crate::cache::AcquireOutcome::Busy(b) => {
17011 panic!("expected the direct acquire to win the lease first: {b:?}")
17012 }
17013 };
17014
17015 let mut state = RunState::new(
17016 repo.clone(),
17017 "main".to_owned(),
17018 head.clone(),
17019 "task".to_owned(),
17020 config,
17021 );
17022 state.candidates = vec![Candidate {
17023 index: 0,
17024 label: 'A',
17025 agent: "alpha".to_owned(),
17026 branch: "does-not-exist".to_owned(),
17027 worktree: repo.clone(),
17028 summary: String::new(),
17029 stat: String::new(),
17030 files: 0,
17031 commits: 0,
17032 empty: false,
17033 failed: None,
17034 verified_noop: None,
17035 duration_ms: 0,
17036 folded: false,
17037 }];
17038 state.tally = Some(Tally {
17039 first_choice: BTreeMap::from([('A', 1)]),
17040 borda: BTreeMap::new(),
17041 winner: 'A',
17042 rankings: 1,
17043 unanimous_initial: true,
17044 deliberated: false,
17045 changed_votes: 0,
17046 unanimous_final: true,
17047 tie_break: None,
17048 judges: 0,
17049 present: 0,
17050 quorum: 0,
17051 met_quorum: true,
17052 uncontested: Some("only candidate A produced a change".to_owned()),
17053 });
17054 state.reviews = vec![ReviewRound {
17058 round: 1,
17059 head: head.clone(),
17060 verified_head: Some(head.clone()),
17061 verified_at: Some(jiff::Timestamp::now()),
17062 reviews: Vec::new(),
17063 e2e: vec![CommandOutcome {
17064 command: format!(
17065 "CARGO_TARGET_DIR='{}' test -f README.md",
17066 cache_dir.display()
17067 ),
17068 code: None,
17069 output_tail: "waiting for the shared build cache".to_owned(),
17070 duration_ms: 0,
17071 resource_blocked: true,
17072 }],
17073 fix: None,
17074 blocking: 1,
17075 answered: 1,
17076 expected: 1,
17077 clean: false,
17078 verify_retried: false,
17079 e2e_deferred: false,
17080 e2e_defer_reason: None,
17081 progressed: false,
17082 vote_split: false,
17083 reconsideration: Vec::new(),
17084 verdict: None,
17085 }];
17086
17087 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17088 let mut runner = Runner {
17089 state,
17090 roles: ResolvedRoles {
17091 implementers: Vec::new(),
17092 judges: Vec::new(),
17093 reviewers: Vec::new(),
17094 fixer: None,
17095 conductor: conductor(),
17096 implementer_roster: Vec::new(),
17097 judge_roster: Vec::new(),
17098 reviewer_roster: Vec::new(),
17099 },
17100 sem: Arc::new(Semaphore::new(1)),
17101 pause: Pause::new(),
17102 interrupt: Pause::new(),
17103 };
17104
17105 runner.review_loop().await.expect("review_loop");
17110
17111 assert_eq!(
17112 runner.state.reviews.len(),
17113 1,
17114 "no new round was started on top of the unresolved one: {:?}",
17115 runner.state.reviews
17116 );
17117 let last = &runner.state.reviews[0];
17118 assert_eq!(
17119 last.e2e_status(),
17120 E2eStatus::ResourceBlocked,
17121 "still contended: {last:?}"
17122 );
17123 assert!(
17124 last.verified_at.expect("still known") > first_attempt_at,
17125 "review_loop must have actually retried the check, not left it exactly as found"
17126 );
17127 assert_ne!(
17128 runner.state.status,
17129 RunStatus::Blocked,
17130 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17131 runner.state.status
17132 );
17133
17134 held.release();
17135 }
17136
17137 #[tokio::test]
17138 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17139 crate::run::pin_test_home();
17140 for status in [RunStatus::Blocked, RunStatus::Gating] {
17141 let mut runner = runner_at(status);
17142 runner.state.config.graph.land = true;
17143 runner.state.gate_ran = true;
17144 runner.state.merge = Some(MergeOutcome {
17145 mode: MergeMode::Pr,
17146 ok: false,
17147 detail: "the fixer produced no commit".into(),
17148 empty: false,
17149 });
17150 let url = "https://example.invalid/o/r/pull/1";
17151 runner.state.pr = Some(crate::run::PrRecord {
17152 url: url.into(),
17153 number: 1,
17154 state: "open".into(),
17155 checks: "red".into(),
17156 round: 1,
17157 rounds: 1,
17158 red_at_merge: Vec::new(),
17159 });
17160 runner.state.save().unwrap();
17161 runner.execute_graph().await.unwrap();
17165 assert_eq!(runner.state.status, RunStatus::Blocked);
17166 assert!(runner.state.land_sync_pending);
17167 assert!(runner.state.events.iter().any(|event|
17168 event.node == "land" && event.message == format!("watching {url}")));
17169 assert_eq!(
17170 runner.state.merge.as_ref().unwrap().detail,
17171 "the fixer produced no commit"
17172 );
17173 }
17174 }
17175
17176 #[tokio::test]
17177 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17178 crate::run::pin_test_home();
17179 let tmp = tempfile::tempdir().expect("tempdir");
17180 let repo = tmp.path().join("repo");
17181 std::fs::create_dir_all(&repo).unwrap();
17182 init_repo(&repo);
17183
17184 let mut config = Config::default();
17185 config.merge.mode = MergeMode::Pr;
17186 config.graph.land = true;
17187 config.graph.land_approval = false;
17188
17189 let mut state = RunState::new(
17190 repo.clone(),
17191 "main".to_owned(),
17192 "deadbeef".to_owned(),
17193 "task".to_owned(),
17194 config,
17195 );
17196 state.candidates = vec![Candidate {
17197 index: 0,
17198 label: 'A',
17199 agent: "alpha".to_owned(),
17200 branch: "does-not-exist".to_owned(),
17201 worktree: repo.clone(),
17202 summary: String::new(),
17203 stat: String::new(),
17204 files: 0,
17205 commits: 0,
17206 empty: false,
17207 failed: None,
17208 verified_noop: None,
17209 duration_ms: 0,
17210 folded: false,
17211 }];
17212 state.tally = Some(Tally {
17213 first_choice: BTreeMap::from([('A', 1)]),
17214 borda: BTreeMap::new(),
17215 winner: 'A',
17216 rankings: 1,
17217 unanimous_initial: true,
17218 deliberated: false,
17219 changed_votes: 0,
17220 unanimous_final: true,
17221 tie_break: None,
17222 judges: 0,
17223 present: 0,
17224 quorum: 0,
17225 met_quorum: true,
17226 uncontested: Some("only candidate A produced a change".to_owned()),
17227 });
17228 state.reviews = vec![ReviewRound {
17229 round: 1,
17230 head: "deadbeef".to_owned(),
17231 verified_head: None,
17232 verified_at: None,
17233 reviews: Vec::new(),
17234 e2e: Vec::new(),
17235 fix: None,
17236 blocking: 0,
17237 answered: 0,
17238 expected: 0,
17239 clean: true,
17240 verify_retried: false,
17241 e2e_deferred: false,
17242 e2e_defer_reason: None,
17243 progressed: false,
17244 vote_split: false,
17245 reconsideration: Vec::new(),
17246 verdict: None,
17247 }];
17248 state.gate = vec![CommandOutcome {
17249 command: "test".to_owned(),
17250 code: Some(0),
17251 output_tail: String::new(),
17252 duration_ms: 0,
17253 resource_blocked: false,
17254 }];
17255 state.gate_ran = true;
17256 state.status = RunStatus::Landing;
17260 state.merge = Some(MergeOutcome {
17261 mode: MergeMode::Pr,
17262 ok: true,
17263 detail: "https://example.invalid/x/y/pull/1".to_owned(),
17264 empty: false,
17265 });
17266
17267 ask_test_home();
17271 state.save().expect("save landing run");
17274 let store = ask::Questions::open();
17275 let q = ask_open_question(&store, &state.id);
17276
17277 let mut runner = Runner {
17278 state,
17279 roles: ResolvedRoles {
17280 implementers: Vec::new(),
17281 judges: Vec::new(),
17282 reviewers: Vec::new(),
17283 fixer: None,
17284 conductor: conductor(),
17285 implementer_roster: Vec::new(),
17286 judge_roster: Vec::new(),
17287 reviewer_roster: Vec::new(),
17288 },
17289 sem: Arc::new(Semaphore::new(1)),
17290 pause: Pause::new(),
17291 interrupt: Pause::new(),
17292 };
17293
17294 runner.execute().await.expect("execute");
17299
17300 assert_eq!(
17301 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17302 Some("https://example.invalid/x/y/pull/1"),
17303 "reentry must not push again or open a second pull request over the \
17304 one `land` is already watching"
17305 );
17306 assert_ne!(
17307 runner.state.status,
17308 RunStatus::Landing,
17309 "land could not actually reach the fake pull request, so it must \
17310 have given up rather than left the run silently parked forever"
17311 );
17312 assert_eq!(runner.state.status, RunStatus::Blocked);
17316 assert!(
17317 store.get(&q.id).unwrap().status.open(),
17318 "Blocked is still alive; settle_questions must have been a no-op here"
17319 );
17320 }
17321
17322 fn state_with_round(round: ReviewRound) -> RunState {
17323 let mut s = RunState::new(
17324 PathBuf::from("/repo"),
17325 "main".to_owned(),
17326 "abc1234".to_owned(),
17327 "add retries".to_owned(),
17328 Config::default(),
17329 );
17330 s.reviews = vec![round];
17331 s
17332 }
17333
17334 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17335 crate::verdict::Finding {
17336 id: id.to_owned(),
17337 severity,
17338 file: None,
17339 line: None,
17340 title: title.to_owned(),
17341 detail: String::new(),
17342 }
17343 }
17344
17345 #[test]
17346 fn pr_body_names_open_findings_and_declined_ones() {
17347 let round = ReviewRound {
17348 round: 2,
17349 head: "deadbee".to_owned(),
17350 verified_head: None,
17351 verified_at: None,
17352 reviews: vec![ReviewRecord {
17353 attempts: 0,
17354 reviewer: 1,
17355 agent: "alpha".to_owned(),
17356 summary: String::new(),
17357 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17358 vote: None,
17359 failed: None,
17360 duration_ms: 0,
17361 }],
17362 e2e: vec![CommandOutcome {
17363 command: "cargo test".to_owned(),
17364 code: Some(0),
17365 output_tail: String::new(),
17366 duration_ms: 0,
17367 resource_blocked: false,
17368 }],
17369 verify_retried: false,
17370 e2e_deferred: false,
17371 e2e_defer_reason: None,
17372 fix: Some(FixRecord {
17373 agent: "alpha".to_owned(),
17374 addressed: Vec::new(),
17375 rejected: vec![crate::verdict::Rejection {
17376 id: "R1-1-1".to_owned(),
17377 why: "not reachable from any caller".to_owned(),
17378 }],
17379 notes: String::new(),
17380 committed: true,
17381 failed: None,
17382 duration_ms: 0,
17383 continuation: None,
17384 }),
17385 blocking: 0,
17386 answered: 1,
17387 expected: 1,
17388 clean: false,
17389 progressed: true,
17390 vote_split: false,
17391 reconsideration: Vec::new(),
17392 verdict: None,
17393 };
17394 let state = state_with_round(round);
17395 let body = pr_message(&state, 'A').body;
17396
17397 assert!(body.contains("add retries"), "the task must still be there");
17398 assert!(body.contains("R2-1-1"), "{body}");
17399 assert!(!body.contains("unused import"), "{body}");
17400 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17401 assert!(
17402 !body.contains("not reachable from any caller"),
17403 "the reason it was declined: {body}"
17404 );
17405 }
17406
17407 #[test]
17408 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17409 let round = ReviewRound {
17410 round: 1,
17411 head: "deadbee".to_owned(),
17412 verified_head: None,
17413 verified_at: None,
17414 reviews: vec![ReviewRecord {
17415 attempts: 0,
17416 reviewer: 1,
17417 agent: "alpha".to_owned(),
17418 summary: String::new(),
17419 findings: Vec::new(),
17420 vote: None,
17421 failed: None,
17422 duration_ms: 0,
17423 }],
17424 e2e: Vec::new(),
17425 verify_retried: false,
17426 e2e_deferred: false,
17427 e2e_defer_reason: None,
17428 fix: None,
17429 blocking: 0,
17430 answered: 1,
17431 expected: 1,
17432 clean: true,
17433 progressed: false,
17434 vote_split: false,
17435 reconsideration: Vec::new(),
17436 verdict: None,
17437 };
17438 let state = state_with_round(round);
17439 let body = pr_message(&state, 'A').body;
17440 assert!(!body.contains("Open review findings"), "{body}");
17441 assert!(!body.contains("Declined"), "{body}");
17442 }
17443
17444 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17445 let mut state = RunState::new(
17446 PathBuf::from("/repo"),
17447 "main".to_owned(),
17448 "abc1234".to_owned(),
17449 instruction.to_owned(),
17450 Config::default(),
17451 );
17452 state.candidates.push(Candidate {
17453 index: 0,
17454 label: 'A',
17455 agent: "alpha".to_owned(),
17456 branch: "magi/x/A".to_owned(),
17457 worktree: PathBuf::from("/wt"),
17458 summary: summary.to_owned(),
17459 stat: String::new(),
17460 files: 1,
17461 commits: 1,
17462 empty: false,
17463 failed: None,
17464 verified_noop: None,
17465 folded: false,
17466 duration_ms: 0,
17467 });
17468 state
17469 }
17470
17471 fn review_state(subjects: &[&str]) -> RunState {
17472 let mut state = state_with_summary(
17473 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17474 "",
17475 );
17476 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17477 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17478 state
17479 }
17480
17481 #[tokio::test]
17484 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17485 ask_test_home();
17486 let tmp = tempfile::tempdir().unwrap();
17487 let repo = tmp.path().join("repo");
17488 std::fs::create_dir_all(&repo).unwrap();
17489 init_repo(&repo);
17490 let origin = tmp.path().join("origin.git");
17491 let g = |dir: &Path, args: &[&str]| {
17492 let out = std::process::Command::new("git")
17493 .args(args)
17494 .current_dir(dir)
17495 .quiet()
17496 .output()
17497 .expect("spawn git");
17498 assert!(
17499 out.status.success(),
17500 "git {args:?}: {}",
17501 String::from_utf8_lossy(&out.stderr)
17502 );
17503 };
17504 g(
17505 tmp.path(),
17506 &[
17507 "clone",
17508 "--bare",
17509 "-q",
17510 repo.to_str().unwrap(),
17511 origin.to_str().unwrap(),
17512 ],
17513 );
17514 g(
17515 &repo,
17516 &["remote", "add", "origin", origin.to_str().unwrap()],
17517 );
17518 let c1 = git::rev_parse(&repo, "main").await.unwrap();
17519
17520 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17522 g(&repo, &["add", "-A"]);
17523 g(
17524 &repo,
17525 &["commit", "-q", "-m", "chore(deps): update a crate"],
17526 );
17527 g(&repo, &["push", "-q", "origin", "main"]);
17528 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17529 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17530 g(&repo, &["add", "-A"]);
17531 g(
17532 &repo,
17533 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17534 );
17535 g(&repo, &["checkout", "-q", "main"]);
17536
17537 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17538 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17539 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17541 assert_eq!(fallback, c1);
17542
17543 let mut state = review_state(&[
17544 "chore(deps): update a crate",
17545 "fix(daemon): apply a chosen action",
17546 ]);
17547 state.repo = repo.clone();
17548 state.base_branch = "main".to_owned();
17549 state.base_commit = c1;
17550 refresh_reviewed_commits(&mut state, "feat/own").await;
17551 assert_eq!(
17552 state.reviewed_commits,
17553 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17554 );
17555 assert_eq!(
17556 review_title(&state).as_deref(),
17557 Some("fix(daemon): apply a chosen action")
17558 );
17559 assert_eq!(
17560 leaked_subjects(&state, "feat/own").await,
17561 Some(vec!["chore(deps): update a crate".to_owned()])
17562 );
17563 state.config.merge.remote = "nowhere".to_owned();
17566 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17567 }
17568
17569 struct SummaryFixture {
17573 _tmp: tempfile::TempDir,
17574 repo: PathBuf,
17575 reply: PathBuf,
17576 runner: Runner,
17577 }
17578
17579 fn summary_fixture() -> SummaryFixture {
17580 ask_test_home();
17581 let tmp = tempfile::tempdir().unwrap();
17582 let repo = tmp.path().join("repo");
17583 std::fs::create_dir_all(&repo).unwrap();
17584 init_repo(&repo);
17585 let origin = tmp.path().join("origin.git");
17586 let g = |args: &[&str]| {
17587 let out = std::process::Command::new("git")
17588 .args(args)
17589 .current_dir(&repo)
17590 .quiet()
17591 .output()
17592 .expect("spawn git");
17593 assert!(
17594 out.status.success(),
17595 "git {args:?}: {}",
17596 String::from_utf8_lossy(&out.stderr)
17597 );
17598 };
17599 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17600 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17601 g(&["push", "-q", "origin", "main"]);
17602 g(&["checkout", "-q", "-b", "feat/own"]);
17603 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17604 g(&["add", "-A"]);
17605 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17606 let reply = tmp.path().join("reply.txt");
17607 std::fs::write(
17608 &reply,
17609 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17610 )
17611 .unwrap();
17612 let mut runner = runner_at(RunStatus::Gating);
17613 let mut state = review_state(&["feat(own): add the own file"]);
17614 state.repo = repo.clone();
17615 state.base_branch = "main".to_owned();
17616 state.candidates[0].branch = "feat/own".to_owned();
17617 state.candidates[0].worktree = repo.clone();
17618 let mut author = spec("alpha");
17619 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17620 author
17621 .env
17622 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17623 state.config.agents = vec![author];
17624 runner.state = state;
17625 SummaryFixture {
17626 _tmp: tmp,
17627 repo,
17628 reply,
17629 runner,
17630 }
17631 }
17632
17633 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17634 let winner = f.runner.state.candidates[0].clone();
17635 f.runner
17636 .guarded_pr_message(&winner, None, true)
17637 .await
17638 .unwrap()
17639 .expect("an English description never parks")
17640 }
17641
17642 fn clean_round_with_minor_findings() -> ReviewRound {
17643 let mut round = passing_round(2, None);
17644 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17645 round.reviews = vec![ReviewRecord {
17646 attempts: 0,
17647 reviewer: 1,
17648 agent: "alpha".into(),
17649 summary: "内部レビュー".into(),
17650 findings: vec![
17651 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17652 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17653 ],
17654 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17655 failed: None,
17656 duration_ms: 0,
17657 }];
17658 round
17659 }
17660
17661 #[tokio::test]
17662 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17663 for review_only in [false, true] {
17664 let mut f = summary_fixture();
17665 if !review_only {
17666 f.runner.state.candidates[0].agent = "alpha".into();
17667 }
17668 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17669 assert!(f.runner.state.reviews[0].clean);
17670 assert!(f.runner.state.reviews[0].fix.is_none());
17671 assert!(f.runner.state.open_findings().is_empty());
17672 assert_eq!(
17673 crate::pr_outcome::required_finding_ids(&f.runner.state),
17674 ["R2-1-1", "R2-1-2"]
17675 );
17676 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17677 let evidence = crate::pr_outcome::summary_evidence(
17678 &f.runner.state,
17679 crate::pr_outcome::FinalHead::At(&head),
17680 );
17681 assert!(
17682 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17683 );
17684 let omitted = posted(&mut f).await;
17686 assert!(omitted.body.contains("No description"));
17687 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17688 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17689 assert!(f.runner.state.change_summary.is_none());
17690 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17691 let described = posted(&mut f).await;
17692 assert!(described.body.contains("improve documentation for callers"));
17693 assert!(described.body.contains("remaining edge case"));
17694 assert!(described.body.contains("## Open review findings"));
17695 assert!(!described.body.contains("未修正"));
17696 }
17697 }
17698
17699 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17700 let mut f = summary_fixture();
17701 if !review_only {
17702 f.runner.state.candidates[0].agent = "alpha".into();
17703 }
17704 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17705 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17706 fixed
17707 .fix
17708 .as_mut()
17709 .unwrap()
17710 .rejected
17711 .push(crate::verdict::Rejection {
17712 id: "R1-1-1".into(),
17713 why: "呼び出されないコードのため辞退".into(),
17714 });
17715 let mut clean = clean_round_with_minor_findings();
17716 clean.verified_head = Some(head.clone());
17717 f.runner.state.reviews = vec![fixed, clean];
17718 ok_gate(&mut f.runner.state, Some(&head));
17719 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17720 f.runner.state.config.agents.push(scripted("beta", reply));
17721 f.runner.state.config.roles.language_judge =
17722 Some(crate::config::AgentChoice::One("beta".into()));
17723 f
17724 }
17725
17726 #[tokio::test]
17727 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17728 for review_only in [false, true] {
17729 let mut f = rewrite_fixture_with_findings(review_only,
17730 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17731 let m = posted(&mut f).await;
17732 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17733 let expected =
17734 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17735 assert!(m.body.contains(&expected));
17736 assert!(m.body.contains("## Review rounds"));
17737 assert!(
17738 m.body
17739 .contains("Gate (`verify.gate`): passed on the final commit")
17740 );
17741 assert!(
17742 m.body
17743 .contains("Tests (`verify.e2e`): passed on the final commit")
17744 );
17745 let c = f.runner.state.change_summary.as_ref().unwrap();
17746 assert!(
17747 m.body
17748 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17749 );
17750 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17751 assert!(crate::pr_outcome::covers_required_findings(
17752 &f.runner.state,
17753 &m.body
17754 ));
17755 assert!(m.body.contains("documentation remains incomplete"));
17756 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17757 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17758 assert!(f.runner.state.github_text.is_none());
17759 }
17760 }
17761
17762 #[tokio::test]
17763 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17764 for review_only in [false, true] {
17765 for reply in [
17766 "TITLE: fix: public description\nA short English description.",
17767 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17768 ] {
17769 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17770 let winner = f.runner.state.candidates[0].clone();
17771 let posted = f
17772 .runner
17773 .guarded_pr_message(&winner, None, true)
17774 .await
17775 .unwrap();
17776 assert!(posted.is_none());
17777 assert!(f.runner.state.parked);
17778 let withheld = f.runner.state.github_text.as_ref().unwrap();
17779 assert!(
17780 withheld.rewrite.is_none(),
17781 "an incomplete rewrite must not be selectable"
17782 );
17783 let q = ask::Questions::open()
17784 .get(withheld.question.as_ref().unwrap())
17785 .unwrap();
17786 assert!(
17787 !q.choices
17788 .contains(&crate::github_text::USE_REWRITE.to_owned())
17789 );
17790 assert!(
17791 q.detail.contains("R1-1-1")
17792 && q.detail.contains("R2-1-1")
17793 && q.detail.contains("R2-1-2")
17794 );
17795 }
17796 }
17797 }
17798
17799 #[tokio::test]
17800 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17801 for review_only in [false, true] {
17802 let mut f = summary_fixture();
17803 if !review_only {
17804 f.runner.state.candidates[0].agent = "alpha".into();
17805 }
17806 f.runner.state.instruction = "内部タスクの説明".into();
17807 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17808 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17809 f.runner.state.reviews = vec![
17810 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17811 round_with_notes(2, Some("二回目の内部修正報告です。")),
17812 passing_round(3, Some(&head)),
17813 ];
17814 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17815 ok_gate(&mut f.runner.state, Some(&head));
17816 let captured = f.repo.join("public-prompt.txt");
17817 let author = &mut f.runner.state.config.agents[0];
17818 author.command = vec![
17819 "sh".into(),
17820 "-c".into(),
17821 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17822 ];
17823 author.env.insert(
17824 "PROMPT_FILE".into(),
17825 captured.to_string_lossy().into_owned(),
17826 );
17827 let m = posted(&mut f).await;
17828 assert!(m.body.contains("Adds `own.txt`"));
17829 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17830 assert!(m.body.contains("R1-1-1"));
17831 assert!(
17832 m.body
17833 .contains("Gate (`verify.gate`): passed on the final commit")
17834 );
17835 assert!(
17836 m.body
17837 .contains("Tests (`verify.e2e`): passed on the final commit")
17838 );
17839 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17840 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17841 let prompt = std::fs::read_to_string(captured).unwrap();
17842 assert!(prompt.contains(&format!(
17843 "git diff {} {head}",
17844 f.runner.state.change_summary.as_ref().unwrap().base
17845 )));
17846 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17847 assert_eq!(
17848 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17849 "二回目の内部修正報告です。"
17850 );
17851 }
17852 }
17853
17854 #[tokio::test]
17855 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17856 for review_only in [false, true] {
17857 let mut f = summary_fixture();
17858 if !review_only {
17859 f.runner.state.candidates[0].agent = "alpha".into();
17860 }
17861 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17862 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17863 std::fs::write(&f.reply, "no usable description").unwrap();
17864 let failed = posted(&mut f).await;
17865 assert!(
17866 failed
17867 .body
17868 .contains("No description of what this change does could be generated")
17869 );
17870 assert!(failed.body.is_ascii());
17871 assert!(f.runner.state.github_text.is_none());
17872 assert!(f.runner.state.change_summary.is_none());
17873 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17874 let evidence = crate::pr_outcome::summary_evidence(
17875 &f.runner.state,
17876 crate::pr_outcome::FinalHead::At(&head),
17877 );
17878 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17879 assert!(
17880 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17881 .exists()
17882 );
17883 f.runner.state =
17884 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17885 std::fs::write(
17886 &f.reply,
17887 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17888 )
17889 .unwrap();
17890 let retried = posted(&mut f).await;
17891 assert!(retried.body.contains("Adds the committed own file."));
17892 assert_eq!(
17893 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17894 "日本語の修正報告"
17895 );
17896 }
17897 }
17898
17899 #[tokio::test]
17900 async fn same_head_verification_and_findings_invalidate_public_prose() {
17901 let mut f = summary_fixture();
17902 posted(&mut f).await;
17903 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17904 let original = f
17905 .runner
17906 .state
17907 .change_summary
17908 .as_ref()
17909 .unwrap()
17910 .evidence
17911 .clone();
17912 ok_gate(&mut f.runner.state, Some(&head));
17913 std::fs::write(
17914 &f.reply,
17915 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17916 )
17917 .unwrap();
17918 let verified = posted(&mut f).await;
17919 assert!(verified.body.contains("now verified"));
17920 assert_ne!(
17921 f.runner.state.change_summary.as_ref().unwrap().evidence,
17922 original
17923 );
17924 let mut round = round_with_notes(1, Some("内部報告"));
17925 round.reviews.push(ReviewRecord {
17926 attempts: 0,
17927 reviewer: 1,
17928 agent: "alpha".into(),
17929 summary: "内部レビュー".into(),
17930 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17931 vote: None,
17932 failed: None,
17933 duration_ms: 0,
17934 });
17935 round
17936 .fix
17937 .as_mut()
17938 .unwrap()
17939 .rejected
17940 .push(crate::verdict::Rejection {
17941 id: "R1-1-1".into(),
17942 why: "呼び出し元から到達できない".into(),
17943 });
17944 f.runner.state.reviews = vec![round];
17945 let omitted = posted(&mut f).await;
17947 assert!(omitted.body.contains("No description"));
17948 assert!(omitted.body.contains("R1-1-1"));
17949 assert!(!omitted.body.contains("到達"));
17950 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17951 let described = posted(&mut f).await;
17952 assert!(described.body.contains("unreachable from callers"));
17953 assert!(described.body.contains("R1-1-1"));
17954 assert!(!described.body.contains("到達"));
17955 let evidence = f
17956 .runner
17957 .state
17958 .change_summary
17959 .as_ref()
17960 .unwrap()
17961 .evidence
17962 .clone();
17963 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17964 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17965 assert!(posted(&mut f).await.body.contains("rationale updated"));
17966 assert_ne!(
17967 f.runner.state.change_summary.as_ref().unwrap().evidence,
17968 evidence
17969 );
17970 f.runner
17972 .state
17973 .change_summary
17974 .as_mut()
17975 .unwrap()
17976 .evidence
17977 .clear();
17978 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17979 assert!(posted(&mut f).await.body.contains("current specification"));
17980 }
17981
17982 #[tokio::test]
17983 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17984 let mut f = summary_fixture();
17985 let m = posted(&mut f).await;
17986 assert_eq!(m.title, "feat(own): add the own file");
17987 assert!(
17988 m.body
17989 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17990 "{}",
17991 m.body
17992 );
17993 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17994 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17995 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17996 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17997 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17998 let kept = f.runner.state.change_summary.as_ref().unwrap();
17999 assert_eq!(
18000 (kept.head.as_str(), kept.base.as_str()),
18001 (head.as_str(), base.as_str())
18002 );
18003 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
18004 }
18005
18006 #[tokio::test]
18007 async fn review_only_summary_receives_the_configured_github_language() {
18008 let mut f = summary_fixture();
18009 f.runner.state.config.graph.github_language = "ja".into();
18010 let captured = f.repo.join("summary-prompt.txt");
18011 let author = &mut f.runner.state.config.agents[0];
18012 author.command = vec![
18013 "sh".into(),
18014 "-c".into(),
18015 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18016 ];
18017 author.env.insert(
18018 "PROMPT_FILE".into(),
18019 captured.to_string_lossy().into_owned(),
18020 );
18021 std::fs::write(
18022 &f.reply,
18023 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18024 )
18025 .unwrap();
18026 let m = posted(&mut f).await;
18027 let prompt = std::fs::read_to_string(captured).unwrap();
18028 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18029 assert!(!prompt.contains("in English"));
18030 assert_eq!(m.title, "feat(own): ファイルを追加");
18031 assert!(m.body.contains("own.txt を追加します。"));
18032 }
18033
18034 #[tokio::test]
18035 async fn review_fixes_come_after_the_whole_change_summary() {
18036 let mut f = summary_fixture();
18037 f.runner.state.reviews = vec![
18038 round_with_notes(1, Some("handled the PATH case")),
18039 round_with_notes(2, None),
18040 ];
18041 let m = posted(&mut f).await;
18042 let summary = m.body.find("## Summary").expect("summary");
18043 let fixes = m.body.find("## Review rounds").expect("review rounds");
18044 assert!(summary < fixes, "{}", m.body);
18045 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18046 assert!(m.body.contains("## Final verification"), "{}", m.body);
18047 }
18048
18049 #[tokio::test]
18050 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18051 let mut f = summary_fixture();
18052 let first = posted(&mut f).await;
18053 let first_head = f.runner.state.change_summary.clone().unwrap().head;
18054 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18056 let again = posted(&mut f).await;
18057 assert_eq!((again.title, again.body), (first.title, first.body));
18058
18059 let g = |args: &[&str]| {
18060 let out = std::process::Command::new("git")
18061 .args(args)
18062 .current_dir(&f.repo)
18063 .quiet()
18064 .output()
18065 .expect("spawn git");
18066 assert!(out.status.success(), "git {args:?}");
18067 };
18068 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18069 g(&["add", "-A"]);
18070 g(&["commit", "-q", "-m", "fix(own): handle more"]);
18071 let second = posted(&mut f).await;
18072 let kept = f.runner.state.change_summary.clone().unwrap();
18073 assert_ne!(kept.head, first_head);
18074 assert!(second.body.contains("Not read yet."), "{}", second.body);
18075 assert!(!second.body.contains("so the first summary is read"));
18076 assert_eq!(second.title, "feat(own): other");
18077 }
18078
18079 #[tokio::test]
18080 async fn an_unavailable_summary_is_stated_and_never_invented() {
18081 let mut f = summary_fixture();
18082 std::fs::write(&f.reply, "no title line here").unwrap();
18083 let m = posted(&mut f).await;
18084 assert!(
18085 m.body
18086 .contains("No description of what this change does could be generated"),
18087 "{}",
18088 m.body
18089 );
18090 assert!(m.body.contains("internal reports remain in local records"));
18091 assert!(f.runner.state.change_summary.is_none());
18092 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18093 let winner = f.runner.state.candidates[0].clone();
18095 let preview = f
18096 .runner
18097 .guarded_pr_message(&winner, None, false)
18098 .await
18099 .unwrap()
18100 .unwrap();
18101 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18102 }
18103
18104 #[test]
18105 fn pr_message_review_single_commit_uses_its_subject() {
18106 let state = review_state(&["feat(nats): per-role user"]);
18107 let m = pr_message(&state, 'A');
18108 assert_eq!(m.title, "feat(nats): per-role user");
18109 assert!(!m.body.contains("Review the work already"), "{}", m.body);
18110 assert!(m.body.contains("## Commits under review"), "{}", m.body);
18111 }
18112
18113 #[test]
18114 fn pr_message_review_multi_commit_takes_the_oldest() {
18115 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18116 let m = pr_message(&state, 'A');
18117 assert_eq!(m.title, "feat: the change");
18118 for s in ["feat: the change", "fix: typo", "fix: again"] {
18119 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18120 }
18121 }
18122
18123 #[test]
18124 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18125 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18126 let mut state = review_state(&[first, "fix: later fixup"]);
18127 state.candidates[0].branch = "機能/ログイン".to_owned();
18128 let m = pr_message(&state, 'A');
18129 assert!(
18130 m.title.starts_with("chore: land candidate A of run"),
18131 "{}",
18132 m.title
18133 );
18134 }
18135 }
18136
18137 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18138 BranchFacts {
18139 commits: commits
18140 .iter()
18141 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18142 .collect(),
18143 stat: stat.to_owned(),
18144 }
18145 }
18146
18147 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18148 let mut r = review_round(true, 0, 1, 1, true, true);
18149 r.round = round;
18150 r.fix = notes.map(|n| FixRecord {
18151 agent: "fixer".to_owned(),
18152 addressed: Vec::new(),
18153 rejected: Vec::new(),
18154 notes: n.to_owned(),
18155 committed: true,
18156 failed: None,
18157 duration_ms: 0,
18158 continuation: None,
18159 });
18160 r
18161 }
18162
18163 #[test]
18164 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18165 let state = review_state(&["ignored"]);
18166 let f = facts(
18167 &[
18168 (
18169 "fix(login): resolve PATH on macOS",
18170 "Login shells skip rc files.",
18171 ),
18172 ("fix: address review", ""),
18173 ],
18174 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18175 );
18176 let m = pr_message_with(&state, 'A', Some(&f));
18177 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18178 assert!(
18179 m.body
18180 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
18181 "{}",
18182 m.body
18183 );
18184 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18185 assert!(m.body.contains("## Diff stat"), "{}", m.body);
18186 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18187 for banned in [
18188 "Review the work already",
18189 "no task statement",
18190 "Original task",
18191 ] {
18192 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18193 }
18194 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18195 }
18196
18197 #[test]
18198 fn pr_message_review_truncates_a_huge_first_commit_body() {
18199 let state = review_state(&["ignored"]);
18200 let f = facts(
18201 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18202 "s",
18203 );
18204 let m = pr_message_with(&state, 'A', Some(&f));
18205 assert!(m.body.len() < 30_000, "{}", m.body.len());
18206 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18207 assert!(
18208 m.body.contains("- ... 1 more commit(s)"),
18209 "{}",
18210 m.body.len()
18211 );
18212 assert!(m.body.ends_with("magi:candidate-a\n"));
18213 }
18214
18215 #[test]
18216 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18217 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18218 assert_eq!(m.title, "feat: x");
18219 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18220 assert!(!m.body.contains("Diff stat"), "{}", m.body);
18221 assert!(!m.body.contains("no task statement"), "{}", m.body);
18222 }
18223
18224 #[test]
18225 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18226 let mut state = review_state(&["日本語の件名"]);
18227 state.candidates[0].branch = "fix/macos-login-path".to_owned();
18228 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18229 state.candidates[0].branch = "機能/ログイン".to_owned();
18230 assert!(
18231 pr_message(&state, 'A')
18232 .title
18233 .starts_with("chore: land candidate A")
18234 );
18235 }
18236
18237 #[test]
18238 fn pr_message_review_fixes_survive_a_clean_final_round() {
18239 let mut state = review_state(&["feat: x"]);
18240 state.reviews = vec![
18241 round_with_notes(1, Some("handled the PATH case")),
18242 round_with_notes(2, None),
18243 ];
18244 let body = pr_message(&state, 'A').body;
18245 assert!(
18246 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18247 "{body}"
18248 );
18249 assert!(body.contains("### Round 1"), "{body}");
18250
18251 state.reviews = vec![
18252 round_with_notes(1, Some("first fix")),
18253 round_with_notes(2, Some("")),
18254 round_with_notes(3, Some("second fix")),
18255 round_with_notes(4, None),
18256 ];
18257 let body = pr_message(&state, 'A').body;
18258 assert!(
18259 body.contains("### Round 1") && !body.contains("first fix"),
18260 "{body}"
18261 );
18262 assert!(
18263 body.contains("### Round 3") && !body.contains("second fix"),
18264 "{body}"
18265 );
18266 assert!(body.contains("### Round 2"), "{body}");
18267 }
18268
18269 const FINAL: &str = "abcdef1234567890";
18270
18271 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18272 let mut r = round_with_notes(round, None);
18273 r.verified_head = verified.map(str::to_owned);
18274 r
18275 }
18276
18277 fn ok_gate(state: &mut RunState, head: Option<&str>) {
18278 state.gate = vec![CommandOutcome {
18279 command: "gate".to_owned(),
18280 code: Some(0),
18281 output_tail: String::new(),
18282 duration_ms: 0,
18283 resource_blocked: false,
18284 }];
18285 state.gate_ran = true;
18286 state.gate_head = head.map(str::to_owned);
18287 }
18288
18289 fn body_at(state: &RunState) -> String {
18290 pr_message_raw(
18291 state,
18292 'A',
18293 None,
18294 crate::pr_outcome::FinalHead::At(FINAL),
18295 ChangeNote::NotAsked,
18296 )
18297 .body
18298 }
18299
18300 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18301
18302 #[test]
18303 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18304 let mut state = review_state(&["feat: x"]);
18305 state.reviews = vec![
18306 round_with_notes(1, Some(SANDBOXED)),
18307 passing_round(2, Some(FINAL)),
18308 ];
18309 ok_gate(&mut state, Some(FINAL));
18310 let body = body_at(&state);
18311 assert!(!body.contains("Handled the PATH case."), "{body}");
18312 assert!(!body.contains("could not commit"), "{body}");
18313 assert!(!body.contains("could not run"), "{body}");
18314 assert!(
18315 body.contains("The fix is committed on the branch"),
18316 "{body}"
18317 );
18318 assert!(!body.contains("sandbox"), "{body}");
18319 assert!(
18320 body.contains("Tests (`verify.e2e`): passed on the final commit"),
18321 "{body}"
18322 );
18323 assert!(
18324 body.contains("Gate (`verify.gate`): passed on the final commit"),
18325 "{body}"
18326 );
18327 assert!(
18329 state.reviews[0]
18330 .fix
18331 .as_ref()
18332 .unwrap()
18333 .notes
18334 .contains("could not commit")
18335 );
18336 }
18337
18338 #[test]
18339 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18340 let mut state = review_state(&["feat: x"]);
18341 let mut r1 = round_with_notes(1, Some(SANDBOXED));
18342 r1.fix.as_mut().unwrap().committed = false;
18343 let mut r2 = passing_round(2, Some(FINAL));
18344 r2.e2e[0].code = Some(1);
18345 state.reviews = vec![r1, r2];
18346 let body = body_at(&state);
18347 assert!(!body.contains("could not commit"), "{body}");
18348 assert!(!body.contains("could not run"), "{body}");
18349 assert!(body.contains("This round produced no commit"), "{body}");
18350 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18351 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18352 assert!(!body.contains("passed"), "{body}");
18353 }
18354
18355 #[test]
18356 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18357 let mut state = review_state(&["feat: x"]);
18358 state.reviews = vec![
18359 round_with_notes(1, Some(SANDBOXED)),
18360 passing_round(2, Some("0123456789abcdef")),
18361 ];
18362 ok_gate(&mut state, Some("0123456789abcdef"));
18363 let body = body_at(&state);
18364 assert!(!body.contains("HTTP tests could not run"), "{body}");
18365 assert!(!body.contains("could not commit"), "{body}");
18367 assert!(
18368 body.contains("passed on an earlier commit (`0123456`)"),
18369 "{body}"
18370 );
18371 assert!(!body.contains("passed on the final commit"), "{body}");
18372
18373 ok_gate(&mut state, None);
18375 let body = body_at(&state);
18376 assert!(
18377 body.contains("passed on a commit that was not recorded"),
18378 "{body}"
18379 );
18380 }
18381
18382 #[test]
18383 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18384 let state = state_with_summary(
18385 "add retries to the client",
18386 "TITLE: feat: retries\n\nDid it.",
18387 );
18388 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18389 assert_eq!(m.title, "feat: retries");
18390 assert!(
18391 m.body.contains("<summary>Original task</summary>"),
18392 "{}",
18393 m.body
18394 );
18395 assert!(!m.body.contains("Commits under review"), "{}", m.body);
18396 assert!(
18397 m.body
18398 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18399 "{}",
18400 m.body
18401 );
18402 }
18403
18404 #[test]
18405 fn pr_message_review_bounds_a_long_english_subject() {
18406 let long = format!("feat: {}", "word ".repeat(100));
18407 let m = pr_message(&review_state(&[&long]), 'A');
18408 assert!(m.title.starts_with("feat: word"), "{}", m.title);
18409 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18410 }
18411
18412 #[test]
18413 fn pr_message_implementation_run_is_unchanged_by_review_support() {
18414 let state = state_with_summary("add retries\n\ndetails", "- did some things");
18415 let m = pr_message(&state, 'A');
18416 assert_eq!(m.title, "add retries");
18417 assert!(m.body.contains("<summary>Original task</summary>"));
18418 assert!(!m.body.contains("Commits under review"));
18419 assert_eq!(landing_subject_source(&state), state.instruction);
18420 }
18421
18422 #[test]
18423 fn review_run_squash_subject_is_the_change_not_the_prompt() {
18424 let state = review_state(&["feat: the change", "fix: typo"]);
18425 let source = landing_subject_source(&state);
18426 assert_eq!(land::merge_subject("", &source), "feat: the change");
18427 assert_eq!(
18428 land::merge_subject("magi: candidate A (uncommitted work)", &source),
18429 "feat: the change"
18430 );
18431 assert_eq!(
18433 land::merge_subject("feat: renamed by hand", &source),
18434 "feat: renamed by hand"
18435 );
18436 let blank = review_state(&["日本語"]);
18437 assert!(
18438 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18439 );
18440 }
18441
18442 #[test]
18443 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18444 let state = review_state(&["feat: the change"]);
18445 let source = landing_subject_source(&state);
18446 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18447 assert_eq!(
18448 land::merge_subject(landing_title(&state, old), &source),
18449 "feat: the change"
18450 );
18451 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18452 let task = state_with_summary("add retries", "");
18453 assert_eq!(landing_title(&task, old), old);
18454 }
18455
18456 #[test]
18457 fn pr_message_describes_the_change_not_the_task() {
18458 let state = state_with_summary(
18459 "今回やってほしいこと: results projector を直す",
18460 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18461 );
18462 let m = pr_message(&state, 'A');
18463 assert_eq!(m.title, "fix(web): batch the runs list reads");
18464 assert!(
18465 m.body.starts_with("## Summary\n\n- reads run.json once"),
18466 "{}",
18467 m.body
18468 );
18469 assert!(!m.body.contains("TITLE:"), "{}", m.body);
18470 let task_at = m.body.find("今回やってほしいこと").unwrap();
18471 let details_at = m.body.find("<details>").unwrap();
18472 assert!(
18473 details_at < task_at,
18474 "the task lives inside <details>: {}",
18475 m.body
18476 );
18477 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18478 assert!(m.body.contains("magi:candidate-a"));
18479 }
18480
18481 #[test]
18482 fn pr_message_falls_back_to_the_task_without_a_title_line() {
18483 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18484 let m = pr_message(&state, 'A');
18485 assert_eq!(m.title, "add retries");
18486 assert!(
18487 m.body.contains("## Summary\n\n- did some things"),
18488 "{}",
18489 m.body
18490 );
18491
18492 let none = RunState::new(
18493 PathBuf::from("/repo"),
18494 "main".to_owned(),
18495 "abc1234".to_owned(),
18496 "add retries".to_owned(),
18497 Config::default(),
18498 );
18499 let m = pr_message(&none, 'A');
18500 assert_eq!(m.title, "add retries");
18501 assert!(!m.body.contains("## Summary"), "{}", m.body);
18502 }
18503
18504 #[test]
18505 fn pr_message_refuses_the_candidate_commit_subject() {
18506 for bad in [
18507 "TITLE: magi: candidate A (uncommitted work)",
18508 "TITLE: chore: stuff (uncommitted work)",
18509 "TITLE: ",
18510 ] {
18511 let state = state_with_summary("add retries", bad);
18512 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18513 }
18514 }
18515
18516 #[test]
18517 fn pr_message_bounds_a_very_long_task_and_title() {
18518 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18519 let state = state_with_summary(&long, "- nothing");
18520 let m = pr_message(&state, 'A');
18521 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18522 assert!(!m.title.contains('\n'));
18523
18524 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18525 let m = pr_message(&state, 'A');
18526 assert!(m.title.starts_with("feat: "));
18527 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18528 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18529 }
18530
18531 fn long_title_of(instruction: &str) -> String {
18532 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18533 }
18534
18535 #[test]
18536 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18537 let first = "Make the landing path keep a readable title for long tasks";
18538 let line = format!(
18539 "{first}. {}",
18540 "And then keep going with more words ".repeat(20)
18541 );
18542 let t = long_title_of(&line);
18543 assert_eq!(t, first);
18544 assert!(!t.starts_with("chore: land"));
18545 }
18546
18547 #[test]
18548 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18549 let line = "word ".repeat(200);
18550 let t = long_title_of(&line);
18551 assert!(t.ends_with("word..."), "{t}");
18552 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18553 }
18554
18555 #[test]
18556 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18557 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18558 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18559 }
18560 }
18561
18562 #[test]
18563 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18564 let mut state = state_with_summary("再試行を追加する", "- nothing");
18565 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18566 state.config.graph.github_language = "ja".into();
18567 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18568 }
18569
18570 #[test]
18571 fn pr_message_title_limit_is_exact() {
18572 let at = "a".repeat(PR_TITLE_MAX);
18573 assert_eq!(long_title_of(&at), at);
18574 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18575 assert!(over.ends_with("..."), "{over}");
18576 assert_eq!(over.chars().count(), PR_TITLE_MAX);
18577 }
18578
18579 #[test]
18580 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18581 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18582 let t = long_title_of(&line);
18583 assert!(t.ends_with("..."), "{t}");
18584 assert!(t.is_ascii(), "{t}");
18585 }
18586
18587 #[test]
18588 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18589 let line = format!(
18590 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18591 "plus more filler words ".repeat(20)
18592 );
18593 let t = long_title_of(&line);
18594 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18595 }
18596
18597 #[test]
18598 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18599 let line = format!("\n\n# {}", "title words ".repeat(40));
18600 let state = state_with_summary(&line, "- nothing");
18601 let m = pr_message(&state, 'A');
18602 assert!(m.title.starts_with("title words"), "{}", m.title);
18603 assert_eq!(
18604 land::merge_subject(&m.title, &landing_subject_source(&state)),
18605 m.title
18606 );
18607 assert_eq!(
18609 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18610 "feat: renamed by hand"
18611 );
18612 }
18613
18614 #[tokio::test]
18615 async fn github_text_rewrite_is_bounded_and_falls_back() {
18616 crate::run::pin_test_home();
18617 for (reply, accepted) in [
18618 (
18619 "TITLE: fix: retries\nAdd retries for failed requests.",
18620 true,
18621 ),
18622 (
18623 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18624 false,
18625 ),
18626 (
18627 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18628 true,
18629 ),
18630 (
18631 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18632 true,
18633 ),
18634 (
18635 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18636 true,
18637 ),
18638 (
18639 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18640 false,
18641 ),
18642 ("TITLE: fix: retries\n", false),
18643 ] {
18644 let dir = tempfile::tempdir().unwrap();
18645 let mut runner = runner_at(RunStatus::Gating);
18646 runner.state = state_with_summary(
18647 "add retries",
18648 "TITLE: fix: retries\n日本語の説明を書きます。",
18649 );
18650 runner.state.repo = dir.path().to_owned();
18651 runner.state.candidates[0].worktree = dir.path().to_owned();
18652 let mut author = spec("alpha");
18653 author.command = vec![
18654 "sh".into(),
18655 "-c".into(),
18656 "printf '%s' \"$REWRITE_REPLY\"".into(),
18657 ];
18658 author.env.insert("REWRITE_REPLY".into(), reply.into());
18659 runner.state.config.agents = vec![author];
18660 seed_public_summary(&mut runner);
18661 let winner = runner.state.candidates[0].clone();
18662 let first = runner
18663 .guarded_pr_message(&winner, None, true)
18664 .await
18665 .unwrap();
18666 let message = if accepted {
18667 first.expect("a passing rewrite never asks")
18668 } else {
18669 assert!(first.is_none());
18671 assert!(runner.state.parked);
18672 let store = ask::Questions::open();
18673 let asked: Vec<_> = store
18674 .list()
18675 .into_iter()
18676 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18677 .collect();
18678 assert_eq!(asked.len(), 1);
18679 assert!(!asked[0].detail.contains("secret"));
18680 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18681 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18682 assert_eq!(
18683 asked[0]
18684 .choices
18685 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18686 rewrite.is_some()
18687 );
18688 store
18691 .update(&asked[0].id, |q| {
18692 q.abandon("test");
18693 Ok(())
18694 })
18695 .unwrap();
18696 let again = runner
18697 .guarded_pr_message(&winner, None, true)
18698 .await
18699 .unwrap()
18700 .expect("an abandoned question falls back");
18701 let n = store
18702 .list()
18703 .into_iter()
18704 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18705 .count();
18706 assert_eq!(n, 1, "asked once per rejected text");
18707 again
18708 };
18709 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18710 assert!(!message.body.contains("secret"));
18711 assert!(!message.body.contains("prefix"));
18712 assert!(!message.title.contains("secret"));
18713 assert_eq!(
18714 message.body.contains("Add retries for failed requests."),
18715 accepted
18716 );
18717 assert_eq!(
18718 runner.state.seats["github-text-rewrite"].turns, 1,
18719 "only one rewrite invocation"
18720 );
18721 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18722 if accepted {
18723 assert!(runner.state.github_text.is_none());
18724 assert!(!runner.state.parked);
18725 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18726 assert!(message.body.contains("## Final verification"));
18727 assert!(message.body.contains("Evidence: final diff"));
18728 assert!(
18729 message
18730 .body
18731 .contains(&format!("magi:run/{}", runner.state.id))
18732 );
18733 }
18734 if !accepted {
18735 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18736 assert!(
18737 message
18738 .body
18739 .contains(&format!("magi:run/{}", runner.state.id))
18740 );
18741 }
18742 }
18743 }
18744
18745 fn scripted(id: &str, reply: &str) -> AgentSpec {
18746 let mut a = spec(id);
18747 a.command = vec![
18748 "sh".into(),
18749 "-c".into(),
18750 "printf '%s' \"$REWRITE_REPLY\"".into(),
18751 ];
18752 a.env.insert("REWRITE_REPLY".into(), reply.into());
18753 a
18754 }
18755
18756 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18757 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18758
18759 fn seed_public_summary(runner: &mut Runner) {
18761 let repo = &runner.state.repo;
18762 let g = |args: &[&str]| {
18763 let out = std::process::Command::new("git")
18764 .args(args)
18765 .current_dir(repo)
18766 .quiet()
18767 .output()
18768 .unwrap();
18769 assert!(out.status.success(), "git {args:?}");
18770 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18771 };
18772 g(&["init", "-q", "-b", "main"]);
18773 g(&[
18774 "-c",
18775 "user.name=Test",
18776 "-c",
18777 "user.email=test@example.invalid",
18778 "commit",
18779 "-q",
18780 "--allow-empty",
18781 "-m",
18782 "feat: fixture",
18783 ]);
18784 let head = g(&["rev-parse", "HEAD"]);
18785 g(&["update-ref", "refs/remotes/origin/main", &head]);
18786 let evidence = crate::pr_outcome::summary_evidence(
18787 &runner.state,
18788 crate::pr_outcome::FinalHead::At(&head),
18789 );
18790 let generated = &runner.state.candidates[0].summary;
18791 runner.state.change_summary = Some(crate::run::ChangeSummary {
18792 base: head.clone(),
18793 head,
18794 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18795 title: summary_title(generated).unwrap(),
18796 body: summary_without_title(generated),
18797 });
18798 }
18799
18800 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18801 let mut runner = runner_at(RunStatus::Gating);
18802 runner.state = state_with_summary("add retries", JA_SUMMARY);
18803 if review_only {
18804 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18805 }
18806 runner.state.repo = dir.to_owned();
18807 runner.state.candidates[0].worktree = dir.to_owned();
18808 runner.state.config.agents = agents;
18809 seed_public_summary(&mut runner);
18810 runner
18811 }
18812
18813 fn gate_questions(runner: &Runner) -> usize {
18814 ask::Questions::open()
18815 .list()
18816 .into_iter()
18817 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18818 .count()
18819 }
18820
18821 #[tokio::test]
18822 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18823 crate::run::pin_test_home();
18824 let dir = tempfile::tempdir().unwrap();
18825 let mut runner = rewrite_runner(
18826 dir.path(),
18827 true,
18828 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18829 );
18830 runner.state.config.roles.language_judge =
18831 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18832 let winner = runner.state.candidates[0].clone();
18833 let msg = runner
18834 .guarded_pr_message(&winner, None, true)
18835 .await
18836 .unwrap()
18837 .expect("an English rewrite posts without asking");
18838 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18839 assert!(msg.body.contains("<details>kept</details>"));
18840 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18841 assert_eq!(gate_questions(&runner), 0);
18842 assert!(runner.state.github_text.is_none());
18843 assert!(!runner.state.seats.contains_key("impl-A"));
18844 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18845 }
18846
18847 #[tokio::test]
18848 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18849 crate::run::pin_test_home();
18850 let dir = tempfile::tempdir().unwrap();
18851 let mut runner = rewrite_runner(
18852 dir.path(),
18853 false,
18854 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18855 );
18856 runner.state.config.roles.language_judge =
18857 Some(crate::config::AgentChoice::One("beta".into()));
18858 runner.state.seats.insert(
18859 "impl-A".into(),
18860 SeatState::new("impl-A", "alpha", runner.state.seed),
18861 );
18862 let winner = runner.state.candidates[0].clone();
18863 let msg = runner
18864 .guarded_pr_message(&winner, None, true)
18865 .await
18866 .unwrap()
18867 .expect("rewritten");
18868 assert!(msg.body.contains("<details>kept</details>"));
18869 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18870 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18871 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18872 }
18873
18874 #[tokio::test]
18875 async fn rewrite_reuses_a_resumable_implementer_session() {
18876 crate::run::pin_test_home();
18877 let dir = tempfile::tempdir().unwrap();
18878 let mut runner = rewrite_runner(
18879 dir.path(),
18880 false,
18881 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18882 );
18883 runner.state.config.roles.language_judge =
18884 Some(crate::config::AgentChoice::One("beta".into()));
18885 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18886 seat.turns = 1;
18887 runner.state.seats.insert("impl-A".into(), seat);
18888 let winner = runner.state.candidates[0].clone();
18889 runner
18890 .guarded_pr_message(&winner, None, true)
18891 .await
18892 .unwrap()
18893 .expect("rewritten");
18894 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18895 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18896 }
18897
18898 #[tokio::test]
18899 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
18900 crate::run::pin_test_home();
18903 let dir = tempfile::tempdir().unwrap();
18904 let mut exhausted = spec("alpha");
18905 exhausted.command = vec!["false".into()];
18906 let mut runner = rewrite_runner(
18907 dir.path(),
18908 false,
18909 vec![exhausted, scripted("beta", EN_REWRITE)],
18910 );
18911 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18913 "alpha".into(),
18914 "beta".into(),
18915 ]));
18916 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18917 seat.turns = 1;
18918 runner.state.seats.insert("impl-A".into(), seat);
18919 let winner = runner.state.candidates[0].clone();
18920 let msg = runner
18921 .guarded_pr_message(&winner, None, true)
18922 .await
18923 .unwrap()
18924 .expect("the role rewrites after the implementer fails");
18925 assert!(msg.body.contains("`src/net.rs`"));
18926 assert_eq!(
18927 runner.state.seats["impl-A"].turns, 2,
18928 "the implementer was tried once"
18929 );
18930 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18931 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18932 }
18933
18934 #[tokio::test]
18935 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18936 crate::run::pin_test_home();
18937 let dir = tempfile::tempdir().unwrap();
18938 let mut broken = spec("bad");
18939 broken.command = vec!["false".into()];
18940 let mut runner = rewrite_runner(
18941 dir.path(),
18942 true,
18943 vec![
18944 broken,
18945 scripted("noformat", "Add retries."),
18946 scripted("good", EN_REWRITE),
18947 ],
18948 );
18949 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18950 "bad".into(),
18951 "noformat".into(),
18952 "good".into(),
18953 ]));
18954 let winner = runner.state.candidates[0].clone();
18955 let msg = runner
18956 .guarded_pr_message(&winner, None, true)
18957 .await
18958 .unwrap()
18959 .expect("the third agent rewrites");
18960 assert!(msg.body.contains("`src/net.rs`"));
18961 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18962 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18963 }
18964
18965 #[tokio::test]
18966 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18967 crate::run::pin_test_home();
18968 for (agents, role, saved) in [
18970 (
18971 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18972 Some("beta"),
18973 true,
18974 ),
18975 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18976 (Vec::new(), None, false),
18977 ] {
18978 let dir = tempfile::tempdir().unwrap();
18979 let mut runner = rewrite_runner(dir.path(), true, agents);
18980 runner.state.config.roles.language_judge =
18981 role.map(|r| crate::config::AgentChoice::One(r.into()));
18982 let winner = runner.state.candidates[0].clone();
18983 let first = runner
18984 .guarded_pr_message(&winner, None, true)
18985 .await
18986 .unwrap();
18987 assert!(first.is_none(), "withheld text parks the run");
18988 assert!(runner.state.parked);
18989 assert_eq!(gate_questions(&runner), 1);
18990 let g = runner.state.github_text.as_ref().unwrap();
18991 assert_eq!(g.rewrite.is_some(), saved);
18992 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18993 }
18994 }
18995
18996 #[tokio::test]
18997 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18998 crate::run::pin_test_home();
18999 let dir = tempfile::tempdir().unwrap();
19000 let mut runner = runner_at(RunStatus::Gating);
19001 runner.state =
19002 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
19003 runner.state.repo = dir.path().to_owned();
19004 runner.state.candidates[0].worktree = dir.path().to_owned();
19005 let mut author = spec("alpha");
19006 author.command = vec![
19007 "sh".into(),
19008 "-c".into(),
19009 "printf '%s' \"$REWRITE_REPLY\"".into(),
19010 ];
19011 author.env.insert(
19012 "REWRITE_REPLY".into(),
19013 format!(
19014 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19015 "English sentence. ".repeat(600)
19016 ),
19017 );
19018 runner.state.config.agents = vec![author];
19019 seed_public_summary(&mut runner);
19020 let winner = runner.state.candidates[0].clone();
19021 assert!(
19022 runner
19023 .guarded_pr_message(&winner, None, true)
19024 .await
19025 .unwrap()
19026 .is_none()
19027 );
19028 let record = runner.state.github_text.clone().unwrap();
19029 let rewrite = record.rewrite.clone().unwrap();
19030 assert_eq!(rewrite.failed, ["body-language"]);
19031 assert!(
19032 rewrite
19033 .body
19034 .ends_with(&format!("magi:run/{}", runner.state.id))
19035 );
19036 let store = ask::Questions::open();
19037 let q = store.get(record.question.as_ref().unwrap()).unwrap();
19038 assert!(
19039 q.choices
19040 .contains(&crate::github_text::USE_REWRITE.to_owned())
19041 );
19042 for label in [
19043 "ORIGINAL",
19044 "FALLBACK",
19045 "REWRITE",
19046 "Rules still failed: body-language",
19047 "truncated",
19048 ] {
19049 assert!(q.detail.contains(label), "{label}");
19050 }
19051 let artifact = std::fs::read_to_string(crate::run::artifact_path(
19052 &runner.state,
19053 &crate::github_text::artifact_name(&record.fingerprint),
19054 ))
19055 .unwrap();
19056 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19057 runner.state =
19059 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19060 store
19061 .update(&q.id, |q| {
19062 q.say("まだ日本語")?;
19063 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19064 })
19065 .unwrap();
19066 assert!(
19067 runner
19068 .guarded_pr_message(&winner, None, true)
19069 .await
19070 .unwrap()
19071 .is_none()
19072 );
19073 let retry = runner.state.github_text.as_ref().unwrap();
19074 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19075 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19076 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19077 assert!(
19078 q.posting_comparison
19079 .as_ref()
19080 .unwrap()
19081 .matches_detail(&q.detail)
19082 );
19083 store
19084 .update(&q.id, |q| {
19085 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19086 })
19087 .unwrap();
19088 let message = runner
19089 .guarded_pr_message(&winner, None, true)
19090 .await
19091 .unwrap()
19092 .unwrap();
19093 assert_eq!(message.title, rewrite.title);
19094 assert_eq!(message.body, rewrite.body);
19095 assert_eq!(
19096 runner.state.github_text.as_ref().unwrap().choice,
19097 Some(GithubTextChoice::Rewrite)
19098 );
19099 let again = runner
19100 .guarded_pr_message(&winner, None, true)
19101 .await
19102 .unwrap()
19103 .unwrap();
19104 assert_eq!(again.title, message.title);
19105 assert_eq!(again.body, message.body);
19106 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19107 assert!(
19108 !runner
19109 .state
19110 .events
19111 .iter()
19112 .any(|e| e.message.contains(&rewrite.title)
19113 || e.message.contains("English sentence."))
19114 );
19115 }
19116
19117 #[tokio::test]
19118 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19119 crate::run::pin_test_home();
19120 let dir = tempfile::tempdir().unwrap();
19121 let mut runner = runner_at(RunStatus::Gating);
19122 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19123 runner.state.repo = dir.path().to_owned();
19124 runner.state.candidates[0].worktree = dir.path().to_owned();
19125 seed_public_summary(&mut runner);
19126 let winner = runner.state.candidates[0].clone();
19127 assert!(
19128 runner
19129 .guarded_pr_message(&winner, None, true)
19130 .await
19131 .unwrap()
19132 .is_none()
19133 );
19134 let record = runner.state.github_text.as_ref().unwrap();
19135 assert!(record.rewrite.is_none());
19136 let mut old = serde_json::to_value(record).unwrap();
19138 old.as_object_mut().unwrap().remove("rewrite");
19139 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19140 assert!(old.rewrite.is_none());
19141 let id = record.question.clone().unwrap();
19142 ask::Questions::open()
19143 .update(&id, |q| {
19144 q.choices.push(crate::github_text::USE_REWRITE.into());
19145 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19146 })
19147 .unwrap();
19148 let message = runner
19149 .guarded_pr_message(&winner, None, true)
19150 .await
19151 .unwrap()
19152 .unwrap();
19153 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19154 assert_eq!(
19155 runner.state.github_text.as_ref().unwrap().choice,
19156 Some(GithubTextChoice::Fallback)
19157 );
19158 }
19159
19160 #[tokio::test]
19161 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19162 crate::run::pin_test_home();
19163 let dir = tempfile::tempdir().unwrap();
19164 let mut runner = runner_at(RunStatus::Gating);
19165 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19166 runner.state.repo = dir.path().to_owned();
19167 runner.state.candidates[0].worktree = dir.path().to_owned();
19168 seed_public_summary(&mut runner);
19169 let winner = runner.state.candidates[0].clone();
19170 assert!(
19171 runner
19172 .guarded_pr_message(&winner, None, true)
19173 .await
19174 .unwrap()
19175 .is_none()
19176 );
19177 let store = ask::Questions::open();
19178 let find = |run: &str| {
19179 store
19180 .list()
19181 .into_iter()
19182 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19183 .unwrap()
19184 };
19185 let run = runner.state.id.clone();
19186 let q = find(&run);
19187 let fp = runner
19188 .state
19189 .github_text
19190 .as_ref()
19191 .unwrap()
19192 .fingerprint
19193 .clone();
19194 let name = crate::github_text::artifact_name(&fp);
19195 let path = crate::run::artifact_path(&runner.state, &name);
19196 assert!(q.detail.contains(&path.display().to_string()));
19197 assert!(q.detail.contains("title-language: title"));
19198 let saved = std::fs::read_to_string(&path).expect("artifact written");
19199 assert!(saved.contains("日本語のタイトル"));
19200 assert!(q.cwd.is_none());
19201 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19202 let say = |id: &str, text: &str, choice: &str| {
19204 store
19205 .update(id, |q| {
19206 q.thread.push(ask::Turn {
19207 who: ask::Who::Operator,
19208 body: text.to_owned(),
19209 at: jiff::Timestamp::now(),
19210 note: None,
19211 });
19212 q.answer(ask::Answer::Choice(choice.to_owned()))
19213 })
19214 .unwrap();
19215 };
19216 say(&q.id, "まだ日本語", "use my text");
19217 assert!(
19218 runner
19219 .guarded_pr_message(&winner, None, true)
19220 .await
19221 .unwrap()
19222 .is_none()
19223 );
19224 let second = find(&run);
19225 assert_ne!(second.id, q.id);
19226 assert!(second.detail.contains("did not pass"));
19227 assert!(second.detail.contains("日本語のタイトル"));
19230 assert!(!second.detail.contains("まだ日本語"));
19231 assert!(second.detail.contains(&path.display().to_string()));
19232 say(&second.id, "fix: retry failed requests", "use my text");
19233 let message = runner
19234 .guarded_pr_message(&winner, None, true)
19235 .await
19236 .unwrap()
19237 .expect("a vetted replacement is posted");
19238 assert_eq!(message.title, "fix: retry failed requests");
19239 assert_eq!(
19240 runner
19241 .state
19242 .github_text
19243 .as_ref()
19244 .unwrap()
19245 .chosen_title
19246 .as_deref(),
19247 Some("fix: retry failed requests")
19248 );
19249 assert!(runner.state.github_text.as_ref().unwrap().resolved);
19250 let again = runner
19252 .guarded_pr_message(&winner, None, true)
19253 .await
19254 .unwrap()
19255 .unwrap();
19256 assert_eq!(again.title, message.title);
19257 assert_eq!(
19258 store
19259 .list()
19260 .into_iter()
19261 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19262 .count(),
19263 2
19264 );
19265 }
19266
19267 #[test]
19268 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19269 let mut state = state_with_summary(
19273 "add retries",
19274 "TITLE: fix(web): batch reads\n- reads run.json once",
19275 );
19276 state.config.graph.language = "ja".to_owned();
19277 let m = pr_message(&state, 'A');
19278 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19279 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19280
19281 let task = "今回やってほしいこと: results projector を直す";
19284 let mut state = state_with_summary(task, "- no title line");
19285 state.config.graph.language = "ja".to_owned();
19286 let m = pr_message(&state, 'A');
19287 assert_eq!(
19288 m.title,
19289 format!("chore: land candidate A of run {}", state.id)
19290 );
19291 assert!(
19292 m.body.contains(&format!(
19293 "<summary>Original task</summary>\n\n{task}\n\n</details>"
19294 )),
19295 "{}",
19296 m.body
19297 );
19298 }
19299
19300 #[test]
19301 fn pr_message_scrubs_home_paths_and_addresses() {
19302 let state = state_with_summary(
19303 "fix it in /Users/someone/src/x",
19304 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19305 );
19306 let m = pr_message(&state, 'A');
19307 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19308 assert!(!m.body.contains(leak), "{}", m.body);
19309 }
19310 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19311 }
19312
19313 #[test]
19314 fn pr_message_survives_a_task_that_closes_details() {
19315 let state = state_with_summary("a </details> b", "TITLE: fix: x");
19316 let m = pr_message(&state, 'A');
19317 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19318 }
19319
19320 #[test]
19321 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19322 let cmd = manual_merge_command(
19323 MergeStyle::Squash,
19324 Path::new("/repo"),
19325 "b",
19326 "fix: \"quoted\" $(x) `y`\n\nbody",
19327 );
19328 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19329 }
19330
19331 #[test]
19332 fn manual_merge_command_matches_the_configured_style() {
19333 let repo = Path::new("/repo");
19334 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19335
19336 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19337 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19338
19339 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19340 assert_eq!(
19341 squash,
19342 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19343 \"Merge magi run 0832 (candidate A)\""
19344 );
19345
19346 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19347 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19348 }
19349
19350 #[test]
19351 fn a_nudge_gets_a_quarter_of_the_budget() {
19352 assert_eq!(retry_budget(secs(1200), true), secs(300));
19354 assert_eq!(retry_budget(secs(3600), true), secs(900));
19355 }
19356
19357 #[test]
19358 fn a_resent_prompt_keeps_the_whole_budget() {
19359 assert_eq!(retry_budget(secs(1200), false), secs(1200));
19362 assert_eq!(retry_budget(secs(60), false), secs(60));
19363 }
19364
19365 #[test]
19366 fn the_floor_never_exceeds_the_original_budget() {
19367 assert_eq!(retry_budget(secs(60), true), secs(60));
19371 assert_eq!(retry_budget(secs(480), true), secs(120));
19372 assert_eq!(retry_budget(secs(0), true), secs(0));
19373 }
19374
19375 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19376 agent::CommandEvidence {
19377 id: "item1".to_owned(),
19378 description: "cargo test".to_owned(),
19379 exit_code,
19380 result_summary: String::new(),
19381 source: "codex".to_owned(),
19382 }
19383 }
19384
19385 #[test]
19386 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19387 assert!(!has_unconfirmed_command(&[]));
19391 }
19392
19393 #[test]
19394 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19395 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19399 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19400 assert!(!has_unconfirmed_command(&[
19401 evidence(Some(0)),
19402 evidence(Some(101))
19403 ]));
19404 }
19405
19406 #[test]
19407 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19408 assert!(has_unconfirmed_command(&[
19409 evidence(Some(0)),
19410 evidence(None)
19411 ]));
19412 }
19413
19414 #[test]
19415 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19416 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19417 assert_eq!(
19418 verified_noop_claim(true, &[], text).as_deref(),
19419 Some("already fixed by b32cfc4, on main.")
19420 );
19421 }
19422
19423 #[test]
19424 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19425 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19428 assert!(verified_noop_claim(false, &[], text).is_none());
19429 }
19430
19431 #[test]
19432 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19433 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19434 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19435 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19437 }
19438
19439 #[test]
19440 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19441 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19442 }
19443
19444 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19447 runner.state.candidates = shape
19448 .iter()
19449 .enumerate()
19450 .map(|(i, &(empty, verified))| Candidate {
19451 index: i,
19452 label: (b'A' + i as u8) as char,
19453 agent: "sonnet".to_owned(),
19454 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19455 worktree: PathBuf::from(format!("/wt/{i}")),
19456 summary: String::new(),
19457 stat: String::new(),
19458 files: 0,
19459 commits: 0,
19460 empty,
19461 failed: None,
19462 verified_noop: verified.map(str::to_owned),
19463 duration_ms: 0,
19464 folded: false,
19465 })
19466 .collect();
19467 }
19468
19469 #[test]
19470 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19471 ask_test_home();
19472 let mut runner = runner_at(RunStatus::Implementing);
19473 set_candidates(
19474 &mut runner,
19475 &[
19476 (true, Some("already on main at b32cfc4")),
19477 (true, Some("same fix, see the existing test")),
19478 ],
19479 );
19480
19481 runner
19482 .after_implement()
19483 .expect("a verified no-op is not an error");
19484
19485 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19486 }
19487
19488 #[test]
19489 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19490 ask_test_home();
19491 let mut runner = runner_at(RunStatus::Implementing);
19492 set_candidates(
19496 &mut runner,
19497 &[(true, Some("already on main at b32cfc4")), (true, None)],
19498 );
19499
19500 let err = runner
19501 .after_implement()
19502 .expect_err("an unverified empty candidate must still fail the run");
19503
19504 assert!(
19505 err.to_string().contains("no candidate produced a change"),
19506 "{err}"
19507 );
19508 assert_eq!(runner.state.status, RunStatus::Failed);
19509 }
19510
19511 #[test]
19512 fn after_implement_still_fails_an_ordinary_all_empty_run() {
19513 ask_test_home();
19514 let mut runner = runner_at(RunStatus::Implementing);
19515 set_candidates(&mut runner, &[(true, None), (true, None)]);
19516
19517 let err = runner
19518 .after_implement()
19519 .expect_err("no candidate declared anything; this is an ordinary failure");
19520
19521 assert!(
19522 err.to_string().contains("no candidate produced a change"),
19523 "{err}"
19524 );
19525 assert_eq!(runner.state.status, RunStatus::Failed);
19526 }
19527}
19528
19529#[derive(Debug)]
19532struct GraftRestoreFailed(String);
19533
19534impl std::fmt::Display for GraftRestoreFailed {
19535 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19536 write!(
19537 f,
19538 "could not restore the winner to its pre-graft commit: {}",
19539 self.0
19540 )
19541 }
19542}
19543
19544impl std::error::Error for GraftRestoreFailed {}
19545
19546#[cfg(test)]
19550mod resource_wait_tests {
19551 use super::*;
19552 use std::collections::BTreeMap;
19553
19554 fn sh(id: &str, script: &str) -> AgentSpec {
19555 AgentSpec {
19556 id: id.to_owned(),
19557 kind: crate::config::AgentKind::Command,
19558 model: None,
19559 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19560 extra_args: Vec::new(),
19561 env: BTreeMap::new(),
19562 prompt_delivery: None,
19563 max_turn_secs: None,
19564 }
19565 }
19566
19567 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19568 SeatJob {
19569 seat: SeatState::new("impl-A", &spec.id, 1),
19570 spec,
19571 cwd: dir.to_path_buf(),
19572 prompt: "do it".to_owned(),
19573 timeout,
19574 allow_write: true,
19575 sessions: false,
19576 artifacts: dir.join("art"),
19577 stem: "impl-A".to_owned(),
19578 handover: None,
19579 }
19580 }
19581
19582 fn state(resource_wait_secs: u64) -> RunState {
19583 let mut config = Config::default();
19584 config.graph.timeout_resource_wait = resource_wait_secs;
19585 RunState::new(
19586 PathBuf::from("/repo"),
19587 "main".to_owned(),
19588 "abc1234".to_owned(),
19589 "task".to_owned(),
19590 config,
19591 )
19592 }
19593
19594 fn hold(cache: &Path) -> crate::cache::Guard {
19595 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19596 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19597 crate::cache::AcquireOutcome::Acquired(g) => g,
19598 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19599 }
19600 }
19601
19602 async fn run(
19603 jobs: Vec<SeatJob>,
19604 sem: Arc<Semaphore>,
19605 cache: Option<&Path>,
19606 st: &mut RunState,
19607 ) -> Vec<(usize, SeatState, AgentOutcome)> {
19608 let prompts = Prompts::default();
19609 let ctx = WaveCtx {
19610 run: "r",
19611 node: "implement",
19612 prompts: &prompts,
19613 cache,
19614 round: None,
19615 carry_seats: false,
19616 pauses: None,
19617 };
19618 wave(jobs, sem, &ctx, st, 0).await
19619 }
19620
19621 #[tokio::test]
19622 async fn verification_cache_wait_preserves_the_command_budget() {
19623 crate::run::pin_test_home();
19624 let tmp = tempfile::tempdir().unwrap();
19625 let cache = tmp.path().join("target");
19626 let guard = hold(&cache);
19627 let release = tokio::spawn(async move {
19628 tokio::time::sleep(Duration::from_millis(100)).await;
19629 guard.release();
19630 });
19631 let mut st = state(10);
19632 let budget = Duration::from_millis(50);
19633 let wait = repository_wait(&st, None);
19634 let (outcomes, _) = with_cache_lease(
19635 &mut st,
19636 Some(&cache),
19637 "gate",
19638 "gate",
19639 tmp.path(),
19640 "abc1234",
19641 budget,
19642 "test gate",
19643 &wait,
19644 |_, available| async move {
19645 assert_eq!(
19646 available, budget,
19647 "contention must not shorten verification"
19648 );
19649 (Vec::new(), false, Vec::new())
19650 },
19651 )
19652 .await
19653 .unwrap();
19654 release.await.unwrap();
19655 assert!(outcomes.is_empty(), "{outcomes:?}");
19656 assert!(
19657 st.events
19658 .iter()
19659 .any(|event| event.message.contains("waiting for"))
19660 );
19661 assert!(st.resource_wait.is_none());
19662 hold(&cache).release();
19664 }
19665
19666 #[tokio::test]
19667 async fn verification_cache_wait_expires_without_starting_commands() {
19668 crate::run::pin_test_home();
19669 let tmp = tempfile::tempdir().unwrap();
19670 let cache = tmp.path().join("target");
19671 let _held = hold(&cache);
19672 let mut st = state(0);
19673 let wait = repository_wait(&st, None);
19674 let (outcomes, _) = with_cache_lease(
19675 &mut st,
19676 Some(&cache),
19677 "gate",
19678 "gate",
19679 tmp.path(),
19680 "abc1234",
19681 Duration::from_secs(60),
19682 "test gate",
19683 &wait,
19684 |_, _| async { panic!("commands must not start without the cache lease") },
19685 )
19686 .await
19687 .unwrap();
19688 assert_eq!(outcomes.len(), 1);
19689 assert!(outcomes[0].resource_blocked);
19690 assert!(st.resource_wait.is_none());
19691 assert!(matches!(
19692 st.resource_block.unwrap().cause,
19693 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19694 ));
19695 }
19696
19697 #[tokio::test]
19698 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19699 crate::run::pin_test_home();
19700 let tmp = tempfile::tempdir().unwrap();
19701 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19702 let cache = tmp.path().join("cache");
19703 let mut state = state(21600);
19704 let wait = repository_wait(&state, None);
19705 let (outcomes, _) = with_cache_lease(
19706 &mut state,
19707 Some(&cache),
19708 "gate",
19709 "gate",
19710 tmp.path(),
19711 "head",
19712 Duration::from_secs(60),
19713 "gate",
19714 &wait,
19715 |_, _| async { panic!("freshness failure must not start verification") },
19716 )
19717 .await
19718 .unwrap();
19719 assert_eq!(outcomes.len(), 1);
19720 assert!(outcomes[0].resource_blocked);
19721 let block = state.resource_block.unwrap();
19722 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19723 assert!(!block.description().contains("timed out"));
19724 }
19725
19726 #[tokio::test]
19727 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19728 crate::run::pin_test_home();
19729 let tmp = tempfile::tempdir().unwrap();
19730 let cache = tmp.path().join("cache");
19731 let _held = hold(&cache);
19732 let mut state = state(21600);
19733 let pause = Pause::new();
19734 let interrupt = Pause::new();
19735 let cause = ParkCause::Operator;
19736 pause.park_for(cause.clone());
19737 let prompts = Prompts::default();
19738 let ctx = WaveCtx {
19739 run: "test",
19740 node: "fix",
19741 prompts: &prompts,
19742 cache: Some(&cache),
19743 round: None,
19744 carry_seats: false,
19745 pauses: Some((&pause, &interrupt)),
19746 };
19747 let result = wave(
19748 vec![job(
19749 sh("writer", "exit 99"),
19750 tmp.path(),
19751 Duration::from_secs(1),
19752 )],
19753 Arc::new(Semaphore::new(1)),
19754 &ctx,
19755 &mut state,
19756 0,
19757 )
19758 .await;
19759 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19760 assert_eq!(result[0].1.turns, 0);
19761 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19762 assert_eq!(
19763 state.resource_block.unwrap().cause,
19764 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19765 );
19766 assert!(state.active.is_empty());
19767 }
19768
19769 #[tokio::test]
19770 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19771 crate::run::pin_test_home();
19772 let tmp = tempfile::tempdir().unwrap();
19773 let cache = tmp.path().join("target-a");
19774 let guard = hold(&cache);
19775 tokio::spawn(async move {
19777 tokio::time::sleep(Duration::from_millis(1500)).await;
19778 guard.release();
19779 });
19780 let mut st = state(60);
19781 let results = run(
19782 vec![job(
19783 sh("alpha", "echo ok"),
19784 tmp.path(),
19785 Duration::from_secs(1),
19786 )],
19787 Arc::new(Semaphore::new(1)),
19788 Some(&cache),
19789 &mut st,
19790 )
19791 .await;
19792 assert!(
19793 matches!(results[0].2, AgentOutcome::Ok(_)),
19794 "the wait must not be charged to the agent"
19795 );
19796 assert!(st.resource_block.is_none());
19797 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19798 assert!(st.resource_wait.is_none() && st.active.is_empty());
19799 }
19800
19801 #[tokio::test]
19802 #[cfg(unix)]
19803 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19804 use std::os::fd::AsRawFd as _;
19805 crate::run::pin_test_home();
19806 let tmp = tempfile::tempdir().unwrap();
19807 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19808 let common = tmp.path().join(".git");
19809 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19810 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19811 assert_eq!(
19812 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19813 0
19814 );
19815 let mut st = state(1);
19816 st.config.coordination = crate::config::test_coordination();
19817 let mut reader = job(
19818 sh("reader", "echo must-not-run"),
19819 tmp.path(),
19820 Duration::from_secs(30),
19821 );
19822 reader.allow_write = false;
19823 let results = run(
19824 vec![reader.clone()],
19825 Arc::new(Semaphore::new(1)),
19826 None,
19827 &mut st,
19828 )
19829 .await;
19830 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19831 assert_eq!(results[0].1.turns, 0);
19832 assert!(FailClass::of(&results[0].2).is_none());
19833 assert_eq!(
19834 st.resource_block.as_ref().unwrap().kind,
19835 "repository coordination"
19836 );
19837 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19838 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19839 drop(external);
19840 let editor = crate::repo_lock::SeatLease::for_checkout(
19841 tmp.path(),
19842 true,
19843 crate::config::test_coordination().as_ref(),
19844 )
19845 .await
19846 .unwrap();
19847 let pause = Pause::new();
19848 let interrupt = Pause::new();
19849 let prompts = Prompts::default();
19850 let ctx = WaveCtx {
19851 run: "r",
19852 node: "review",
19853 prompts: &prompts,
19854 cache: None,
19855 round: None,
19856 carry_seats: false,
19857 pauses: Some((&pause, &interrupt)),
19858 };
19859 let (results, ()) = tokio::join!(
19860 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19861 async {
19862 tokio::time::sleep(Duration::from_millis(50)).await;
19863 pause.park();
19864 }
19865 );
19866 assert!(
19867 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19868 );
19869 assert_eq!(results[0].1.turns, 0);
19870 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19871 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19872 drop(editor);
19873 }
19874
19875 #[tokio::test]
19876 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19877 crate::run::pin_test_home();
19878 let tmp = tempfile::tempdir().unwrap();
19879 let cache = tmp.path().join("target-b");
19880 let _held = hold(&cache);
19881 let mut st = state(1);
19882 let results = run(
19883 vec![job(
19884 sh("alpha", "echo ok"),
19885 tmp.path(),
19886 Duration::from_secs(1),
19887 )],
19888 Arc::new(Semaphore::new(1)),
19889 Some(&cache),
19890 &mut st,
19891 )
19892 .await;
19893 let (_, seat, out) = &results[0];
19894 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19895 assert!(wave_blocked(&results));
19896 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19897 assert_eq!(seat.turns, 0, "nothing was asked");
19898 assert!(
19899 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19900 "no agent was spawned"
19901 );
19902 assert!(FailClass::of(out).is_none(), "not a failure class");
19903 let block = st.resource_block.as_ref().expect("recorded for the resume");
19904 assert_eq!(block.kind, "shared build cache");
19905 assert!(matches!(
19906 block.cause,
19907 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
19908 ));
19909 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19910 assert!(st.active.is_empty());
19911 }
19912
19913 #[tokio::test]
19914 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19915 crate::run::pin_test_home();
19916 let tmp = tempfile::tempdir().unwrap();
19917 let mut st = state(1);
19918 let results = run(
19919 vec![job(
19920 sh("alpha", "echo ok"),
19921 tmp.path(),
19922 Duration::from_secs(1),
19923 )],
19924 Arc::new(Semaphore::new(0)),
19925 None,
19926 &mut st,
19927 )
19928 .await;
19929 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19930 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19931 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19932 }
19933
19934 #[tokio::test]
19935 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19936 crate::run::pin_test_home();
19937 let tmp = tempfile::tempdir().unwrap();
19938 let mut st = state(60);
19939 let results = run(
19940 vec![job(
19941 sh("alpha", "sleep 30"),
19942 tmp.path(),
19943 Duration::from_secs(1),
19944 )],
19945 Arc::new(Semaphore::new(1)),
19946 None,
19947 &mut st,
19948 )
19949 .await;
19950 let out = &results[0].2;
19951 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19952 let class = FailClass::of(out).expect("a failure");
19953 assert_eq!(class, FailClass::Timeout);
19954 assert!(should_hand_over(None, &class), "existing handover policy");
19955 assert!(st.resource_block.is_none());
19956 }
19957}