1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530enum FixStep {
532 Resource,
533 Recorded { report_lost: bool },
534}
535
536struct FixClaim {
553 path: PathBuf,
554}
555
556impl FixClaim {
557 fn acquire(dir: &Path) -> Result<Self> {
558 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559 let path = dir.join("fix.lock");
560 let retry = crate::lock_file::Retry::new(None);
561 match Self::create(&path, retry) {
562 Ok(claim) => Ok(claim),
563 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564 if Self::reclaim_if_dead(&path) {
565 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566 } else {
567 bail!(
568 "another repair or `magi fix` is already running for this run ({} exists)",
569 path.display()
570 )
571 }
572 }
573 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574 }
575 }
576
577 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578 let mut f = retry.create(path)?;
579 use std::io::Write as _;
580 writeln!(f, "{}", std::process::id())?;
582 Ok(Self {
583 path: path.to_owned(),
584 })
585 }
586
587 fn reclaim_if_dead(path: &Path) -> bool {
591 let dead = std::fs::read_to_string(path)
592 .ok()
593 .and_then(|body| body.trim().parse::<u32>().ok())
594 .is_some_and(|pid| !crate::proc::pid_alive(pid));
595 dead && std::fs::remove_file(path).is_ok()
596 }
597}
598
599impl Drop for FixClaim {
600 fn drop(&mut self) {
601 let _ = std::fs::remove_file(&self.path);
602 }
603}
604
605struct DriverExecution<'a> {
608 runner: &'a mut Runner,
609 driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613 fn drop(&mut self) {
614 self.driver.release_local(&mut self.runner.state);
615 }
616}
617
618impl Runner {
619 pub async fn start(
621 repo: &Path,
622 instruction: String,
623 config: Config,
624 origin: Origin,
625 ) -> Result<Self> {
626 Self::start_naming(repo, instruction, "", config, origin).await
627 }
628
629 pub async fn start_naming(
632 repo: &Path,
633 instruction: String,
634 _also_scan: &str,
635 config: Config,
636 origin: Origin,
637 ) -> Result<Self> {
638 let repo = git::toplevel(repo).await?;
639 let missing = agent::missing_programs(&config.agents);
640 if !missing.is_empty() {
641 bail!(
642 "these agent programs are not on PATH: {}. Fix the roster in \
643 magi.toml or install them.",
644 missing.join(", ")
645 );
646 }
647 let base_branch =
648 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649 .await?;
650 let fetched_base = resolve_base(
651 &repo,
652 &base_branch,
653 &config.merge.remote,
654 config.coordination.as_ref(),
655 )
656 .await?;
657 let base_commit = fetched_base.commit.clone();
658 let roles = config.resolve_roles()?;
659 let max_parallel = config.graph.max_parallel.max(1);
660 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661 state.fetched_base = Some(fetched_base);
665 state.origin = Some(origin);
666 state.reserve()?;
667 state.event("start", format!("run {} created", state.id));
668 state.save()?;
669 Ok(Self {
670 state,
671 roles,
672 sem: Arc::new(Semaphore::new(max_parallel)),
673 pause: Pause::new(),
674 interrupt: Pause::new(),
675 })
676 }
677
678 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692 Self::review_taking_over(repo, branch, config, None, origin).await
693 }
694
695 pub async fn review_taking_over(
702 repo: &Path,
703 branch: &str,
704 config: Config,
705 takeover: Option<crate::handover::Takeover>,
706 origin: Origin,
707 ) -> Result<Self> {
708 let repo = git::toplevel(repo).await?;
709 let missing = agent::missing_programs(&config.agents);
710 if !missing.is_empty() {
711 bail!(
712 "these agent programs are not on PATH: {}. Fix the roster in \
713 magi.toml or install them.",
714 missing.join(", ")
715 );
716 }
717 let base_branch =
718 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719 .await?;
720 if base_branch == branch {
721 bail!("`{branch}` is the base branch; there is nothing to review against");
722 }
723 let fetched_base = resolve_base(
724 &repo,
725 &base_branch,
726 &config.merge.remote,
727 config.coordination.as_ref(),
728 )
729 .await?;
730 let base_commit = fetched_base.commit.clone();
731
732 let roles = config.resolve_roles()?;
733 let max_parallel = config.graph.max_parallel.max(1);
734 let mut state = RunState::new(
735 repo.clone(),
736 base_branch,
737 base_commit.clone(),
738 String::new(),
739 config,
740 );
741 state.fetched_base = Some(fetched_base);
742 state.origin = Some(origin);
743 state.reserve()?;
744
745 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752 earlier: Vec::new(),
753 home: crate::run::home(),
754 choice: None,
755 task_instruction: None,
756 });
757 let wait = repository_wait(&state, None);
758 let coordination_config = state.config.coordination.clone();
759 let coordination =
760 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761 .await?;
762 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763 if let Some(released) = &released {
764 state.event(
765 "release",
766 format!(
767 "took `{branch}` over from run {}: its worktree was released: {}",
768 crate::run::short_of(&released.old_id),
769 released.audit
770 ),
771 );
772 }
773 if let Some(choice) = takeover.choice.as_ref()
777 && let Err(e) =
778 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779 .await
780 {
781 if let Some(released) = &released {
782 released.restore(&repo, branch).await;
783 }
784 return Err(e.context("applying the owner's answer about the diverged branch"));
785 }
786 for id in takeover.earlier.iter().rev() {
792 if let Ok(old) = RunState::load_under(id, &takeover.home)
793 && let Some(head) = old.pr_head.clone()
794 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795 {
796 state.pr_head = Some(head);
797 break;
798 }
799 }
800 drop(coordination);
801 let opened = Self::open_review(
802 &repo,
803 branch,
804 state,
805 roles,
806 max_parallel,
807 base_commit,
808 &takeover,
809 )
810 .await;
811 if opened.is_err()
812 && let Some(released) = &released
813 {
814 let _coordination = crate::repo_lock::SeatLease::operation_with(
815 &repo,
816 coordination_config.as_ref(),
817 &wait,
818 )
819 .await?;
820 released.restore(&repo, branch).await;
821 }
822 opened
823 }
824
825 pub async fn recover_pr(
827 repo: &Path,
828 task: &str,
829 selection: crate::pr_recovery::Selection,
830 mut config: Config,
831 ) -> Result<Self> {
832 selection.configure(&mut config)?;
833 selection.check_task(task, &selection.instruction)?;
834 let base_branch =
835 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836 .await?;
837 let fetched_base = resolve_base(
838 repo,
839 &base_branch,
840 &config.merge.remote,
841 config.coordination.as_ref(),
842 )
843 .await?;
844 let base_commit = fetched_base.commit.clone();
845 selection
846 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847 .await?;
848 Box::pin(selection.verify_source(repo)).await?;
849 if let Some(proof) = &selection.source_proof {
850 anyhow::ensure!(
851 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852 "source rebase base is outside the fetched landing base"
853 );
854 }
855 let roles = config.resolve_roles()?;
856 let max_parallel = config.graph.max_parallel.max(1);
857 let mut state = RunState::new(
858 repo.to_path_buf(),
859 base_branch,
860 base_commit.clone(),
861 selection.instruction.clone(),
862 config,
863 );
864 state.fetched_base = Some(fetched_base);
865 state.origin = Some(crate::run::Origin::queue(task));
866 state.pr_head = Some(selection.branch.clone());
867 state.pr_recovery = Some(selection.clone());
868 state.reserve()?;
869 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870 state.discard_reserved();
872 return Err(e);
873 }
874 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875 }
876
877 async fn open_selected(
878 repo: &Path,
879 state: RunState,
880 roles: ResolvedRoles,
881 max_parallel: usize,
882 base_commit: String,
883 selection: crate::pr_recovery::Selection,
884 ) -> Result<Self> {
885 let mut state = state;
886 let branch = state.branch_for('A');
887 let wait = repository_wait(&state, None);
888 let prepared = async {
889 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890 git::git(repo, &["branch", &branch, selection.source()]).await?;
891 drop(coordination);
892 anyhow::Ok(())
893 }
894 .await;
895 if let Err(e) = prepared {
896 state.discard_reserved();
899 return Err(e);
900 }
901 state.event(
902 "recovery",
903 format!(
904 "selected implementation {} for {} (public lease {})",
905 selection.source(),
906 selection.url,
907 selection.head
908 ),
909 );
910 let takeover = crate::handover::Takeover {
911 earlier: Vec::new(),
912 home: crate::run::home(),
913 choice: None,
914 task_instruction: Some(selection.instruction.clone()),
915 };
916 if let Err(e) = state.save() {
917 state.discard_reserved();
918 return Err(e);
919 }
920 let id = state.id.clone();
921 let opened = Self::open_review(
922 repo,
923 &branch,
924 state,
925 roles,
926 max_parallel,
927 base_commit,
928 &takeover,
929 )
930 .await;
931 match opened {
932 Ok(runner) => Ok(runner),
933 Err(e) => {
934 if let Ok(mut partial) = RunState::load(&id) {
935 partial.status = RunStatus::Blocked;
936 partial.driver_exited = true;
937 partial.event(
938 "recovery",
939 format!("startup failed; retained source and candidate {branch}: {e:#}"),
940 );
941 partial.save()?;
942 }
943 Err(crate::pr_recovery::StartupFailed {
944 run: id,
945 cause: format!("{e:#}"),
946 }
947 .into())
948 }
949 }
950 }
951
952 async fn inherit(
957 repo: &Path,
958 branch: &str,
959 base_commit: &str,
960 takeover: &crate::handover::Takeover,
961 ) -> Option<crate::recovery::RecoveryRecord> {
962 use crate::recovery::{Relation, assess};
963 let newest = takeover.earlier.last()?;
964 let mut chosen = None;
965 for id in takeover.earlier.iter().rev() {
966 if let Ok(old) = RunState::load_under(id, &takeover.home)
967 && !old.reviews.is_empty()
968 && (old.winner().is_some_and(|w| w.branch == branch)
969 || old.pr_head.as_deref() == Some(branch))
970 {
971 chosen = Some(old);
972 break;
973 }
974 }
975 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976 .await
977 .unwrap_or_default();
978 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979 Some(last) if last.head == tip => Relation::Same,
980 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981 _ => Relation::Diverged,
982 };
983 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984 Some(assess(from, chosen.as_ref(), base_commit, relation))
985 }
986
987 async fn open_review(
990 repo: &Path,
991 branch: &str,
992 mut state: RunState,
993 roles: ResolvedRoles,
994 max_parallel: usize,
995 base_commit: String,
996 takeover: &crate::handover::Takeover,
997 ) -> Result<Self> {
998 let wait = repository_wait(&state, None);
999 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000 if state.pr_recovery.is_none() {
1001 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002 }
1003 let start = review_base(
1006 repo,
1007 &state.config.merge.remote,
1008 &state.base_branch,
1009 &base_commit,
1010 branch,
1011 )
1012 .await;
1013 let log = git::log_oneline(repo, &start, branch)
1014 .await
1015 .unwrap_or_default();
1016 let log_text = if log.trim().is_empty() {
1017 "(no commit messages)"
1018 } else {
1019 log.trim()
1020 };
1021 let mut instruction = format!(
1022 "Review the work already on branch `{branch}`. There is no task \
1023 statement: what the change claims to do is whatever its commits \
1024 say.\n\n{log_text}"
1025 );
1026 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030 if let Some(task) = takeover
1031 .task_instruction
1032 .as_deref()
1033 .filter(|t| !t.trim().is_empty())
1034 && rec.reviewed_head.is_some()
1035 {
1036 instruction = format!(
1037 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038 task.trim()
1039 );
1040 rec.criteria_inherited = true;
1041 }
1042 state.event("recovery", rec.summary());
1043 state.recovery = Some(rec);
1044 }
1045 if let Some(selected) = &state.pr_recovery {
1046 instruction = selected.instruction.clone();
1047 }
1048 state.task_instruction = takeover.task_instruction.clone();
1049 state.instruction = instruction;
1050 state.reviewed_commits = Some(
1051 git::subjects(repo, &start, branch)
1052 .await
1053 .unwrap_or_default(),
1054 );
1055
1056 for id in &takeover.earlier {
1059 if let Ok(old) = RunState::load_under(id, &takeover.home)
1060 && old.repo == state.repo
1061 && (old.pr_head.as_deref() == Some(branch)
1062 || old.winner().is_some_and(|w| w.branch == branch))
1063 {
1064 for hold in old.branch_owner_holds {
1065 if !state.branch_owner_holds.contains(&hold) {
1066 state.event(
1067 "approval-policy",
1068 format!(
1069 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070 hold.control.key(),
1071 hold.file,
1072 hold.head
1073 ),
1074 );
1075 state.branch_owner_holds.push(hold);
1076 }
1077 }
1078 }
1079 }
1080 let reviewed_head = git::rev_parse(repo, branch).await?;
1081 state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083 let worktree = state.worktree_root().join("under-review");
1086 if let Some(parent) = worktree.parent() {
1087 tokio::fs::create_dir_all(parent).await.ok();
1088 }
1089 let path = worktree.to_string_lossy().to_string();
1090 git::git(repo, &["worktree", "add", &path, branch])
1091 .await
1092 .with_context(|| {
1093 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094 })?;
1095
1096 let setup = match ensure_setup_config(&mut state, repo).await {
1097 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098 Err(e) => Err(e),
1099 };
1100 if let Err(e) = setup {
1101 git::worktree_remove_clean(repo, &worktree).await.ok();
1102 return Err(e);
1103 }
1104 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105 .await
1106 .unwrap_or(0);
1107 if commits == 0 {
1108 git::worktree_remove_clean(repo, &worktree).await.ok();
1109 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110 }
1111 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112 .await
1113 .map(|f| f.len())
1114 .unwrap_or(0);
1115 if files == 0
1116 && let (Ok(head_tree), Ok(base_tree)) = (
1117 git::tree_of(&worktree, "HEAD").await,
1118 git::tree_of(&worktree, &base_commit).await,
1119 )
1120 && head_tree == base_tree
1121 {
1122 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123 git::worktree_remove_clean(repo, &worktree).await.ok();
1124 bail!(
1125 "`{branch}` at {} has a tree identical to base {}; this usually means \
1126 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127 against `{}/{branch}`) rather than an empty change",
1128 short(&head),
1129 short(&base_commit),
1130 state.config.merge.remote
1131 );
1132 }
1133 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134 .await
1135 .unwrap_or_default();
1136
1137 state.candidates.push(Candidate {
1138 index: 0,
1139 label: 'A',
1140 agent: EXISTING_BRANCH.to_owned(),
1143 branch: branch.to_owned(),
1144 worktree,
1145 summary: String::new(),
1146 stat,
1147 files,
1148 commits,
1149 empty: false,
1150 failed: None,
1151 verified_noop: None,
1152 duration_ms: 0,
1153 folded: false,
1154 });
1155 state.tally = Some(Tally {
1156 first_choice: BTreeMap::from([('A', 0)]),
1157 borda: BTreeMap::new(),
1158 winner: 'A',
1159 rankings: 0,
1160 unanimous_initial: false,
1161 deliberated: false,
1162 changed_votes: 0,
1163 unanimous_final: false,
1164 tie_break: None,
1165 judges: 0,
1169 present: 0,
1170 quorum: 0,
1171 met_quorum: true,
1172 uncontested: Some("review-only run: nothing competed".to_owned()),
1173 });
1174 state.status = RunStatus::Reviewing;
1175 state.prep_complete = Some(true);
1176 state.event(
1177 "start",
1178 format!(
1179 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180 state.id
1181 ),
1182 );
1183 state.save()?;
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub fn resume(id: &str) -> Result<Self> {
1195 let state = RunState::load(id)?;
1196 crate::pr_head::ensure_owner(&state)?;
1197 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198 bail!(
1199 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200 );
1201 }
1202 if let Some(to) = &state.released_to {
1203 bail!(
1204 "run {} cannot be resumed: its worktree was released to run {}",
1205 state.short(),
1206 crate::run::short_of(to)
1207 );
1208 }
1209 let roles = state.config.resolve_roles()?;
1210 if let Some(id) = &state.config.graph.selected_implementer
1211 && state.candidates.iter().any(|c| c.agent != *id)
1212 {
1213 bail!(
1214 "replacement implementer pin does not match the saved candidates; task must stay held"
1215 );
1216 }
1217 let max_parallel = state.config.graph.max_parallel.max(1);
1218 Ok(Self {
1219 state,
1220 roles,
1221 sem: Arc::new(Semaphore::new(max_parallel)),
1222 pause: Pause::new(),
1223 interrupt: Pause::new(),
1224 })
1225 }
1226
1227 pub async fn execute(&mut self) -> Result<()> {
1234 if let Some(selection) = &self.state.pr_recovery
1235 && (selection.source_proof.is_some() || selection.source_run.is_some())
1236 {
1237 Box::pin(selection.verify_source(&self.state.repo)).await?;
1238 }
1239 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242 || self.state.land_approval.is_some()
1243 || self.state.merge_decision.is_some()
1244 || !self.state.merge_repairs.is_empty()
1245 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246 {
1247 let claim = FixClaim::acquire(&self.state.dir())?;
1248 let disk = RunState::load(&self.state.id)?;
1249 let checkpoint = |s: &RunState| {
1250 (
1251 s.merge_repairs.len(),
1252 s.merge_repairs
1253 .last()
1254 .map(|r| (r.question.clone(), r.phase)),
1255 s.land_approval
1256 .as_ref()
1257 .map(|a| (a.question.clone(), a.head.clone())),
1258 )
1259 };
1260 if checkpoint(&disk) != checkpoint(&self.state) {
1261 bail!(
1262 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263 );
1264 }
1265 Some(claim)
1266 } else {
1267 None
1268 };
1269 let mut execution = self.claim_driver().await?;
1270 let runner = &mut *execution.runner;
1271 let wait = runner.repository_wait();
1272 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275 let result = match result {
1276 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277 record_repository_block(&mut runner.state, &e);
1278 runner.park_here()?;
1279 Ok(())
1280 }
1281 other => other,
1282 };
1283 execution.driver.finish(&mut runner.state).await?;
1284 let ended = if result.is_err() {
1285 Some({
1286 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1287 if let Err(error) = &result {
1288 notice.message = format!(
1289 "Run {} stopped with an error: {}",
1290 runner.state.id,
1291 error.to_string().lines().next().unwrap_or("unknown error")
1292 );
1293 }
1294 notice
1295 })
1296 } else {
1297 crate::notices::run_ended(&runner.state)
1298 };
1299 if let Some(notice) = ended {
1300 crate::notices::raise(notice);
1301 }
1302 result
1303 }
1304
1305 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1306 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1307 }
1308
1309 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1310 let wait = self.repository_wait();
1311 coordinate_operation(&mut self.state, cwd, &wait).await
1312 }
1313
1314 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1316 let owner = match crate::pr_head::owner_lock_wait(
1317 &crate::run::home(),
1318 &self.state.id,
1319 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1320 )
1321 .await
1322 {
1323 Ok(owner) => owner,
1324 Err(error) => {
1325 let detail = format!("worktree maintenance or ownership update: {error:#}");
1326 self.state.resource_block = Some(crate::run::ResourceBlock {
1330 cause: crate::run::ResourceBlockCause::Other,
1331 node: "start".into(),
1332 kind: "worktree maintenance".into(),
1333 detail: detail.clone(),
1334 at: Timestamp::now(),
1335 });
1336 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1337 }
1338 };
1339 let disk = RunState::load(&self.state.id)?;
1340 crate::pr_head::ensure_owner(&disk)?;
1341 let claimed =
1342 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1343 anyhow::ensure!(
1348 disk.liveness(claimed) != crate::run::Liveness::Live,
1349 "run already has a live driver; wait for its execution to end"
1350 );
1351 if let Some(to) = &disk.released_to {
1352 bail!(
1353 "run {} cannot continue: its worktree was released to run {}",
1354 self.state.short(),
1355 crate::run::short_of(to)
1356 );
1357 }
1358 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1359 let mut execution = DriverExecution {
1360 runner: self,
1361 driver,
1362 };
1363 let _owner = owner;
1366 let state = &mut execution.runner.state;
1367 state.parked = false;
1372 state.resource_wait = None;
1373 state.resource_block = None;
1374 state.clear_active();
1381 let pid = std::process::id();
1396 state.driver_pid = Some(pid);
1397 state.driver_started_at = crate::proc::process_started_at(pid);
1398 state.driver_exited = false;
1399 state.driver_claimed = Some(true);
1400 state.begin_pass();
1401 execution.driver.bind(state);
1402 state.save()?;
1403 execution.driver.record(state)?;
1404 Ok(execution)
1405 }
1406
1407 async fn execute_graph(&mut self) -> Result<()> {
1408 self.state.configure_recovery_resume()?;
1409 self.persist_candidates().await?;
1412 if self
1413 .state
1414 .merge_decision
1415 .as_ref()
1416 .is_some_and(|decision| !decision.closed)
1417 && self.state.status != RunStatus::Landing
1418 {
1419 if crate::merge_decision::resumable_hold(&self.state) {
1420 self.state.merge = Some(MergeOutcome {
1421 mode: MergeMode::Pr,
1422 ok: false,
1423 detail: self
1424 .state
1425 .merge_decision
1426 .as_ref()
1427 .unwrap()
1428 .target
1429 .pr
1430 .clone(),
1431 empty: false,
1432 });
1433 self.run_land().await?;
1434 self.settle_questions();
1435 return Ok(());
1436 }
1437 bail!(
1438 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1439 );
1440 }
1441 if self.state.config.graph.selected_implementer.is_some()
1442 && self.state.status == RunStatus::Blocked
1443 && self.state.tally.is_none()
1444 && self.state.candidate_preservation.is_empty()
1445 && self.state.interrupted_implementations.is_empty()
1446 {
1447 bail!(
1448 "selected implementation is held; explicitly release the owning task to retry it"
1449 );
1450 }
1451
1452 if self.state.merge_repairs.last().is_some_and(|r| {
1453 matches!(
1454 r.phase,
1455 crate::run::MergeRepairPhase::Requested
1456 | crate::run::MergeRepairPhase::Fixing
1457 | crate::run::MergeRepairPhase::Reviewing
1458 )
1459 }) {
1460 if self.repair_merge_review().await? {
1461 self.run_land().await?;
1462 }
1463 self.settle_questions();
1464 return Ok(());
1465 }
1466 if self.state.pr_publication_pending {
1469 self.merge().await?;
1470 if self.resource_halt()? {
1471 return Ok(());
1472 }
1473 self.state.save()?;
1474 self.settle_questions();
1475 return Ok(());
1476 }
1477 if self.state.status == RunStatus::Stalled {
1490 if self.recover_stall().await? {
1491 self.finish_after_tally().await?;
1492 } else {
1493 self.state.save()?;
1495 }
1496 return Ok(());
1497 }
1498 if self.state.status == RunStatus::Landing
1508 || crate::handover::landing_checkpoint(&self.state)
1509 {
1510 self.state.land_sync_pending = true;
1511 self.state.save()?;
1512 self.run_land().await?;
1513 self.settle_questions();
1518 return Ok(());
1519 }
1520 if self.state.github_text.is_some()
1525 && self.state.merge.is_none()
1526 && self.state.status == RunStatus::Gating
1527 {
1528 self.merge().await?;
1529 if self.resource_halt()? {
1530 return Ok(());
1531 }
1532 self.state.save()?;
1533 return Ok(());
1534 }
1535 if let Err(e) = self.prep().await {
1536 if e.is::<crate::repo_lock::ResourceBlocked>() {
1537 return Err(e);
1538 }
1539 self.state.status = RunStatus::Blocked;
1540 self.state
1541 .prep_failure
1542 .get_or_insert_with(|| format!("{e:#}"));
1543 self.state
1544 .event("prep", format!("preparation stopped: {e:#}"));
1545 self.state.save()?;
1546 return Err(e);
1547 }
1548 if self.park_here()? {
1549 return Ok(());
1550 }
1551 self.advise().await?;
1552 if self.park_here()? {
1553 return Ok(());
1554 }
1555 self.implement().await?;
1556 if self.park_here()? {
1557 return Ok(());
1558 }
1559 if self.state.status == RunStatus::VerifiedNoop {
1563 return Ok(());
1564 }
1565 self.judge().await?;
1566 if self.park_here()? {
1567 return Ok(());
1568 }
1569 self.deliberate().await?;
1570 if self.park_here()? {
1571 return Ok(());
1572 }
1573 self.vote().await?;
1574 if self.park_here()? {
1575 return Ok(());
1576 }
1577 self.tally()?;
1578 if self.state.status == RunStatus::Stalled {
1583 self.state.save()?;
1587 return Ok(());
1588 }
1589 self.finish_after_tally().await?;
1590 Ok(())
1591 }
1592
1593 fn stop_for_resource(&mut self) -> Result<()> {
1598 if self.pause.parked() || self.interrupt.parked() {
1601 self.park_here()?;
1602 return Ok(());
1603 }
1604 if self.state.resource_block.is_some() {
1605 self.state.status = RunStatus::Blocked;
1606 }
1607 self.state.save()
1608 }
1609
1610 fn resource_halt(&mut self) -> Result<bool> {
1612 if self.state.resource_block.is_none() {
1613 return Ok(false);
1614 }
1615 self.stop_for_resource()?;
1616 Ok(true)
1617 }
1618
1619 fn park_here(&mut self) -> Result<bool> {
1626 let parking = self.pause.parked() || self.interrupt.parked();
1627 if !parking && self.state.resource_block.is_some() {
1628 self.stop_for_resource()?;
1629 return Ok(true);
1630 }
1631 if !self.pause.parked() && !self.interrupt.parked() {
1636 return Ok(false);
1637 }
1638 if self.state.parked {
1639 return Ok(true);
1640 }
1641 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1642 if let Some(block) = &mut self.state.resource_block
1643 && matches!(
1644 block.cause,
1645 crate::run::ResourceBlockCause::Cancelled { .. }
1646 )
1647 {
1648 block.cause = crate::run::ResourceBlockCause::Cancelled {
1649 park: cause.clone(),
1650 };
1651 }
1652 let why = match cause.as_ref().map(ParkCause::label) {
1653 Some(reason) => format!(
1654 "parked after `{}` ({reason}) — resume to carry on from here",
1655 self.state.status.as_str()
1656 ),
1657 None => format!(
1658 "parked after `{}` — resume to carry on from here",
1659 self.state.status.as_str()
1660 ),
1661 };
1662 self.state.event("park", why);
1663 self.state
1664 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1665 self.state.parked = true;
1666 self.state.save()?;
1667 Ok(true)
1668 }
1669
1670 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1672 if self.state.prep_failure.is_none() {
1673 return Ok(());
1674 }
1675 let failed_at = self
1676 .state
1677 .events
1678 .iter()
1679 .rev()
1680 .find(|event| event.node == "prep")
1681 .map_or(self.state.updated_at, |event| event.at);
1682 anyhow::ensure!(
1683 approval_at > failed_at,
1684 "preparation retry requires a new explicit release after the failure"
1685 );
1686 self.state.prep_failure = None;
1687 self.state.prep_complete = Some(false);
1688 self.state
1689 .event("prep-retry", "operator approved retry of preparation");
1690 Ok(())
1691 }
1692
1693 pub fn on_pause(&mut self, pause: Pause) {
1695 self.pause = pause;
1696 }
1697
1698 pub fn watch_interrupt(&mut self, pause: Pause) {
1704 self.interrupt = pause;
1705 }
1706
1707 fn settle_questions(&mut self) {
1727 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1728 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1729 }
1730 }
1731
1732 async fn finish_after_tally(&mut self) -> Result<()> {
1735 self.graft().await?;
1738 if self.resource_halt()? {
1739 return Ok(());
1740 }
1741 self.fold_losers().await?;
1742 self.sync_to_base().await?;
1747 if self.resource_halt()? {
1748 return Ok(());
1749 }
1750 if self.state.status == RunStatus::AlreadyInBase {
1751 return Ok(());
1752 }
1753 self.review_loop().await?;
1754 if self.resource_halt()? {
1755 return Ok(());
1756 }
1757 self.sync_to_base().await?;
1758 if self.resource_halt()? {
1759 return Ok(());
1760 }
1761 if self.state.status == RunStatus::AlreadyInBase {
1762 return Ok(());
1763 }
1764 self.gate().await?;
1765 self.merge().await?;
1766 if self.resource_halt()? {
1767 return Ok(());
1768 }
1769 self.state.save()?;
1770 Ok(())
1771 }
1772
1773 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1780 let result = match ensure_setup_config(&mut self.state, repo).await {
1781 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1782 Err(e) => Err(e),
1783 };
1784 if let Err(e) = result {
1785 git::worktree_remove_clean(repo, wt).await.ok();
1786 if in_prep {
1787 self.rollback_prep(repo).await;
1788 }
1789 self.state.status = RunStatus::Blocked;
1790 self.state.event(
1791 "setup",
1792 format!("worktree setup failed in {}: {e:#}", wt.display()),
1793 );
1794 self.state.save()?;
1795 return Err(e);
1796 }
1797 Ok(())
1798 }
1799
1800 async fn rollback_prep(&mut self, repo: &Path) {
1802 let root = self.state.worktree_root();
1803 for c in std::mem::take(&mut self.state.candidates) {
1804 if matches!(
1805 git::worktree_remove_clean(repo, &c.worktree).await,
1806 Ok(true)
1807 ) {
1808 git::branch_delete(repo, &c.branch).await.ok();
1809 } else {
1810 self.state.candidates.push(c);
1811 }
1812 }
1813 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1814 for j in 1..=seats {
1815 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1816 .await
1817 .ok();
1818 }
1819 for k in 1..=self.state.config.graph.advisors {
1820 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1821 .await
1822 .ok();
1823 }
1824 }
1825
1826 async fn prep(&mut self) -> Result<()> {
1827 if let Some(reason) = &self.state.prep_failure {
1828 bail!(
1829 "candidate preparation previously failed: {reason}; correct the source before retrying"
1830 );
1831 }
1832 if self
1833 .state
1834 .prep_complete
1835 .unwrap_or(!self.state.candidates.is_empty())
1836 {
1837 return Ok(());
1838 }
1839 let repo = self.state.repo.clone();
1840 let _coordination = self.coordinate(&repo).await?;
1843 let base = self.state.base_commit.clone();
1844 let plan = self.state.sources.plan(&repo, &base).await?;
1845 self.rollback_prep(&repo).await;
1846 if !self.state.candidates.is_empty() {
1847 bail!(
1848 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1849 );
1850 }
1851 self.state.prep_complete = Some(false);
1852 self.state.status = RunStatus::Prep;
1853 self.state.save()?;
1854 let repo = self.state.repo.clone();
1855 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1856 let root = self.state.worktree_root();
1857 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1858
1859 let hooks_dir = self.state.dir().join("hooks");
1862 if self.state.config.blind.commit_msg_hook {
1863 std::fs::create_dir_all(&hooks_dir)
1864 .with_context(|| format!("create {}", hooks_dir.display()))?;
1865 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1866 let path = hooks_dir.join("commit-msg");
1867 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1868 make_executable(&path)?;
1869 git::acquire_worktree_config(&repo).await?;
1877 self.state.enabled_worktree_config = true;
1878 }
1879
1880 ensure_setup_config(&mut self.state, &repo).await?;
1883
1884 for (index, (spec, label)) in self
1885 .roles
1886 .implementers
1887 .clone()
1888 .into_iter()
1889 .zip(labels)
1890 .enumerate()
1891 {
1892 let branch = self.state.branch_for(label);
1893 let worktree = root.join(format!("cand-{label}"));
1894 self.state.event(
1898 "prep-intent",
1899 serde_json::json!({
1900 "branch": branch,
1901 "worktree": worktree,
1902 "start": start,
1903 "picks": plan.picks,
1904 })
1905 .to_string(),
1906 );
1907 self.state.save()?;
1908 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1909 if self.state.config.blind.commit_msg_hook {
1910 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1911 }
1912 git::local_exclude(&worktree, "/.magi/").await?;
1913 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1914 git::worktree_remove_clean(&repo, &worktree).await.ok();
1917 git::branch_delete(&repo, &branch).await.ok();
1918 self.rollback_prep(&repo).await;
1919 self.state.status = RunStatus::Blocked;
1920 self.state
1921 .event("prep", format!("worktree setup failed: {e:#}"));
1922 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1923 self.state.save()?;
1924 return Err(e);
1925 }
1926 self.state.candidates.push(Candidate {
1927 index,
1928 label,
1929 agent: spec.id.clone(),
1930 branch,
1931 worktree,
1932 summary: String::new(),
1933 stat: String::new(),
1934 files: 0,
1935 commits: 0,
1936 empty: false,
1937 failed: None,
1938 verified_noop: None,
1939 duration_ms: 0,
1940 folded: false,
1941 });
1942 self.state.save()?;
1943 for pick in &plan.picks {
1944 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1945 {
1946 self.state.status = RunStatus::Blocked;
1947 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1948 self.state
1949 .event("prep", format!("cannot apply referenced commit: {e}"));
1950 self.state.save()?;
1951 return Err(e);
1952 }
1953 }
1954 }
1955
1956 for j in 1..=self.roles.judges.len() {
1957 let wt = root.join(format!("judge-{j}"));
1958 if !wt.exists() {
1959 git::worktree_add_detached(&repo, &wt, &base).await?;
1960 self.setup_seat_worktree(&repo, &wt, true).await?;
1961 }
1962 }
1963
1964 if self.state.config.graph.advise {
1972 for k in 1..=self.state.config.graph.advisors {
1973 let wt = root.join(format!("advisor-{k}"));
1974 if !wt.exists() {
1975 git::worktree_add_detached(&repo, &wt, &base).await?;
1976 self.setup_seat_worktree(&repo, &wt, true).await?;
1977 }
1978 }
1979 }
1980
1981 let authors: Vec<&str> = self
1986 .roles
1987 .implementers
1988 .iter()
1989 .map(|a| a.id.as_str())
1990 .collect();
1991 let overlap: Vec<String> = self
1992 .roles
1993 .judges
1994 .iter()
1995 .enumerate()
1996 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1997 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1998 .collect();
1999 if !overlap.is_empty() {
2000 let note = format!(
2001 "{} also authored a candidate; blind, but the panel is less \
2002 independent than {} distinct agents would be",
2003 overlap.join(", "),
2004 self.roles.judges.len()
2005 );
2006 self.state.event("prep", note);
2007 }
2008
2009 self.state.event(
2010 "prep",
2011 format!(
2012 "{} candidates, {} judges, base {} ({})",
2013 self.state.candidates.len(),
2014 self.roles.judges.len(),
2015 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2016 self.state.base_branch
2017 ),
2018 );
2019 self.state.prep_complete = Some(true);
2020 self.state.status = RunStatus::Implementing;
2021 self.state.save()?;
2022 Ok(())
2023 }
2024
2025 async fn advise(&mut self) -> Result<()> {
2058 let implement_untouched = self
2059 .state
2060 .candidates
2061 .iter()
2062 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2063 if !self.state.config.graph.advise || self.state.advise_attempted {
2064 return Ok(());
2065 }
2066 if !implement_untouched {
2067 self.state.event(
2068 "advise",
2069 "skipping the design-deliberation stage: at least one \
2070 candidate already shows implementation progress, so this \
2071 run is past the point the stage exists to run before"
2072 .to_owned(),
2073 );
2074 self.state.advise_attempted = true;
2075 self.state.save()?;
2076 return Ok(());
2077 }
2078 let run_id = self.state.id.clone();
2079 let prompts = self.state.config.prompts.clone();
2080 let instruction = self.state.instruction.clone();
2081 let language = self.state.config.graph.language.clone();
2082 let root = self.state.worktree_root();
2083 let n = self.state.config.graph.advisors;
2084 let where_recorded = self.state.dir().join("run.json");
2085
2086 let seats = match self.state.config.advisors() {
2087 Ok(seats) if !seats.is_empty() => seats,
2088 Ok(_) => {
2089 self.state.event(
2090 "advise",
2091 format!(
2092 "[graph] advisors is 0; skipping the design-deliberation \
2093 stage and continuing without a synthesis brief (see {})",
2094 where_recorded.display()
2095 ),
2096 );
2097 self.state.advise_attempted = true;
2098 self.state.save()?;
2099 return Ok(());
2100 }
2101 Err(e) => {
2102 self.state.event(
2103 "advise",
2104 format!(
2105 "could not resolve advisor seats ({e:#}); continuing \
2106 without a design-deliberation brief (see {})",
2107 where_recorded.display()
2108 ),
2109 );
2110 self.state.advise_attempted = true;
2111 self.state.save()?;
2112 return Ok(());
2113 }
2114 };
2115
2116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2117 let artifacts = agent::artifacts_dir(&self.state.dir());
2118 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2119
2120 let mut jobs = Vec::new();
2121 for (i, spec) in seats.iter().cloned().enumerate() {
2122 let seat_key = format!("advisor-{}", i + 1);
2123 let seat = self.seat(&seat_key, &spec.id);
2124 jobs.push(SeatJob {
2125 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2126 spec,
2127 seat,
2128 cwd: worktrees[i % worktrees.len()].clone(),
2129 timeout,
2130 allow_write: false,
2131 sessions: false,
2132 artifacts: artifacts.clone(),
2133 stem: seat_key,
2134 handover: None,
2135 });
2136 }
2137
2138 self.state.event(
2139 "advise",
2140 format!(
2141 "{} advisor seat(s) sketching a design in parallel",
2142 jobs.len()
2143 ),
2144 );
2145 let mut quota_losses = Vec::new();
2146 let cache = self.state.config.cache_dir();
2147 let ctx = WaveCtx {
2148 carry_seats: false,
2149 pauses: Some((&self.pause, &self.interrupt)),
2150 run: &run_id,
2151 node: "advise",
2152 prompts: &prompts,
2153 cache: cache.as_deref(),
2154 round: None,
2155 };
2156 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2157 let results = ask_json_wave::<Proposal>(
2158 jobs,
2159 Arc::clone(&self.sem),
2160 self.state.config.graph.retries,
2161 &advisor_roster,
2162 &ctx,
2163 &mut quota_losses,
2164 &mut self.state,
2165 &|p: &Proposal| p.validate(),
2166 )
2167 .await;
2168 if self.state.resource_block.is_some() {
2169 return self.stop_for_resource();
2170 }
2171 self.state.quota.extend(quota_losses);
2172
2173 let mut records = Vec::with_capacity(results.len());
2174 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2175 let agent_id = seat.agent.clone();
2176 self.state.seats.insert(seat.key.clone(), seat);
2177 match res {
2178 Ok((proposal, out)) => {
2179 self.state
2180 .event("advise", format!("advisor-{} proposed a design", i + 1));
2181 records.push(advise::AdvisorRecord::proposed(
2182 i + 1,
2183 agent_id,
2184 proposal,
2185 out.duration_ms,
2186 ));
2187 }
2188 Err(e) => {
2189 self.state.event(
2190 "advise",
2191 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2192 );
2193 records.push(advise::AdvisorRecord::failed(
2194 i + 1,
2195 agent_id,
2196 e.to_string(),
2197 ));
2198 }
2199 }
2200 }
2201
2202 let mut advice = advise::Advice {
2203 records,
2204 synthesis: None,
2205 };
2206 if advice.proposals().is_empty() {
2207 self.state.event(
2208 "advise",
2209 "no advisor produced a usable proposal; continuing without a \
2210 synthesis brief"
2211 .to_owned(),
2212 );
2213 } else {
2214 match self
2215 .synthesize_brief(
2216 &advice,
2217 &instruction,
2218 &language,
2219 &worktrees[0],
2220 &artifacts,
2221 &run_id,
2222 &prompts,
2223 cache.as_deref(),
2224 )
2225 .await
2226 {
2227 Ok(Some(text)) => {
2228 self.state.event(
2229 "advise",
2230 "synthesized a design brief for the implementer".to_owned(),
2231 );
2232 advice.synthesis = Some(text);
2233 }
2234 Ok(None) => {
2235 self.state.event(
2236 "advise",
2237 "the synthesis seat produced nothing usable; continuing \
2238 without a design brief"
2239 .to_owned(),
2240 );
2241 }
2242 Err(e) => {
2243 if e.is::<crate::repo_lock::ResourceBlocked>() {
2244 return Err(e);
2245 }
2246 self.state.event(
2247 "advise",
2248 format!("could not synthesize a design brief: {e:#}"),
2249 );
2250 }
2251 }
2252 }
2253 advise::apply_reflection(&mut advice);
2254
2255 self.state.advice = Some(advice);
2256 self.state.advise_attempted = true;
2257 self.state.save()?;
2258 Ok(())
2259 }
2260
2261 #[allow(clippy::too_many_arguments)]
2273 async fn synthesize_brief(
2274 &mut self,
2275 advice: &advise::Advice,
2276 instruction: &str,
2277 language: &str,
2278 cwd: &Path,
2279 artifacts: &Path,
2280 run_id: &str,
2281 prompts: &Prompts,
2282 cache: Option<&Path>,
2283 ) -> Result<Option<String>> {
2284 let chain = agent::pick_chain(
2285 &self.state.config.agents,
2286 self.state.config.roles.synthesizer.as_ref(),
2287 &agent::installed,
2288 "synthesizer",
2289 )?;
2290 let proposals = advice.proposals();
2291 let mut prompt = prompt::with_overlay(
2292 prompt::synthesize_brief(instruction, &proposals, language),
2293 prompts.overlay("advise"),
2294 );
2295 if cache.is_some() {
2296 prompt.push('\n');
2301 prompt.push_str(&prompt::build_cache_note("advise", false));
2302 }
2303 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2304 let mut last = None;
2309 for (n, spec) in chain.iter().enumerate() {
2310 if n > 0 {
2311 self.state
2312 .event("advise", format!("synthesis falling back to {}", spec.id));
2313 }
2314 let mut seat = self.seat("advise-synthesis", &spec.id);
2315 let wait = self.repository_wait();
2316 let outcome = wait
2317 .scope(agent::invoke(
2318 spec,
2319 &mut seat,
2320 &Invocation {
2321 coordination: self.state.config.coordination.as_ref(),
2322 fetched_base: None,
2323 cwd,
2324 prompt: &prompt,
2325 timeout,
2326 allow_write: false,
2327 unsandboxed: false,
2328 sessions: false,
2329 artifacts,
2330 stem: &if n == 0 {
2331 "advise-synthesis".to_owned()
2332 } else {
2333 format!("advise-synthesis-{}", spec.id)
2334 },
2335 run: run_id,
2336 node: "advise",
2337 cache_dir: None,
2338 attachments: &[],
2339 writable: &[],
2340 },
2341 ))
2342 .await;
2343 if outcome.is_ok() {
2344 self.state.seats.insert(seat.key.clone(), seat);
2345 }
2346 let advance = agent::chain_advances(&outcome);
2347 last = Some(outcome);
2348 if !advance {
2349 break;
2350 }
2351 }
2352 let out = last.expect("a chain holds at least one agent")?;
2355 if !out.usable() {
2356 return Ok(None);
2357 }
2358 let text =
2359 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2360 Ok((!text.trim().is_empty()).then_some(text))
2361 }
2362
2363 async fn implement(&mut self) -> Result<()> {
2366 self.persist_candidates().await?;
2367 let run_id = self.state.id.clone();
2372 let prompts = self.state.config.prompts.clone();
2373 let todo: Vec<usize> = self
2374 .state
2375 .candidates
2376 .iter()
2377 .enumerate()
2378 .filter(|(_, c)| {
2379 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2380 && c.failed.is_none()
2381 && !c.empty
2382 })
2383 .map(|(i, _)| i)
2384 .collect();
2385 if todo.is_empty() {
2386 return self.after_implement();
2387 }
2388 self.state.status = RunStatus::Implementing;
2389
2390 let language = self.state.config.graph.language.clone();
2391 let github_language = self.state.config.graph.github_language.clone();
2392 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2393 let sessions = self.state.config.graph.sessions;
2394 let artifacts = agent::artifacts_dir(&self.state.dir());
2395 let brief = self
2399 .state
2400 .advice
2401 .as_ref()
2402 .and_then(|a| a.synthesis.as_deref())
2403 .map(str::to_owned);
2404 let attachments = self.state.attachments.clone();
2405
2406 let mut jobs = Vec::new();
2407 for &i in &todo {
2408 let (index, label, worktree) = {
2409 let c = &self.state.candidates[i];
2410 (c.index, c.label, c.worktree.clone())
2411 };
2412 let spec = self.roles.implementers[index].clone();
2413 let seat_key = format!("impl-{label}");
2414 let seat = self.seat(&seat_key, &spec.id);
2415 let instruction = seeded_instruction(&self.state);
2416 jobs.push(SeatJob {
2417 spec,
2418 seat,
2419 prompt: prompt::implement_in(
2420 &instruction,
2421 &worktree.to_string_lossy(),
2422 &language,
2423 &github_language,
2424 brief.as_deref(),
2425 &attachments,
2426 ),
2427 cwd: worktree,
2428 timeout,
2429 allow_write: true,
2430 sessions,
2431 artifacts: artifacts.clone(),
2432 stem: format!("impl-{label}"),
2433 handover: None,
2434 });
2435 }
2436
2437 self.state.event(
2438 "implement",
2439 format!("{} candidates in parallel", jobs.len()),
2440 );
2441 let mut sent = jobs.clone();
2447 let cache = self.state.config.cache_dir();
2448 let ctx = WaveCtx {
2449 carry_seats: false,
2450 pauses: Some((&self.pause, &self.interrupt)),
2451 run: &run_id,
2452 node: "implement",
2453 prompts: &prompts,
2454 cache: cache.as_deref(),
2455 round: None,
2456 };
2457 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2458 if !wave_blocked(&results) {
2459 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2460 .await;
2461 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2462 .await?;
2463 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2464 .await;
2465 }
2466 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2467
2468 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2469 && results
2470 .iter()
2471 .any(|(_, _, out)| FailClass::of(out).is_some());
2472 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2473 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2474 if self
2478 .state
2479 .candidate_preservation
2480 .contains_key(&self.state.candidates[i].label)
2481 {
2482 self.state.seats.insert(seat.key.clone(), seat);
2483 }
2484 continue;
2485 }
2486 let seat_key = seat.key.clone();
2487 let agent = seat.agent.clone();
2496 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2497 self.state.seats.insert(seat.key.clone(), seat);
2498 let label = self.state.candidates[i].label;
2499 self.state.interrupted_implementations.remove(&label);
2500
2501 let (summary, duration, failed, verified_claim) = match out {
2502 AgentOutcome::Ok(o) => {
2503 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2504 let failed = (!o.usable()).then(|| {
2505 if o.timed_out {
2506 "agent timed out".to_owned()
2507 } else {
2508 format!(
2509 "agent exited with {}",
2510 crate::notices::exit_status(o.exit_code)
2511 )
2512 }
2513 });
2514 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2515 (text, o.duration_ms, failed, verified_claim)
2516 }
2517 AgentOutcome::Dropped(o) => {
2523 let why = o
2524 .dropped
2525 .as_ref()
2526 .map(|d| d.why.as_str())
2527 .unwrap_or("the CLI ended the stream without delivering its answer");
2528 (
2529 String::new(),
2530 o.duration_ms,
2531 Some(format!("the CLI dropped the stream ({why})")),
2532 None,
2533 )
2534 }
2535 AgentOutcome::Quota(o) => {
2536 self.state.quota.push(QuotaLoss {
2537 seat: seat_key,
2538 node: "implement".to_owned(),
2539 at: Timestamp::now(),
2540 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2541 });
2542 (
2543 String::new(),
2544 o.duration_ms,
2545 Some("rate limited (quota); produced no change".to_owned()),
2546 None,
2547 )
2548 }
2549 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2550 (String::new(), 0, Some(e), None)
2551 }
2552 };
2553
2554 let c = &mut self.state.candidates[i];
2557 if !exhausted_the_fallback_chain {
2558 c.agent = agent;
2559 }
2560 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2561 c.duration_ms = duration;
2562 c.failed = failed;
2563 c.verified_noop = verified_claim;
2564 self.state
2565 .candidate_preservation
2566 .insert(label, "awaiting persistence".to_owned());
2567 }
2568 self.state.save()?;
2569 self.persist_candidates().await?;
2570
2571 if resource_blocked {
2572 return self.stop_for_resource();
2573 }
2574 if selected_failed {
2575 self.state.status = RunStatus::Blocked;
2576 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2577 self.state.save()?;
2578 bail!(
2579 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2580 );
2581 }
2582 self.after_implement()
2583 }
2584
2585 async fn persist_candidates(&mut self) -> Result<()> {
2587 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2588 for label in labels {
2589 let i = self
2590 .state
2591 .candidates
2592 .iter()
2593 .position(|c| c.label == label)
2594 .context("pending candidate is missing")?;
2595 let c = self.state.candidates[i].clone();
2596 let interrupted = self.state.interrupted_implementations.contains(&label);
2597 let result = async {
2598 let wait = crate::repo_lock::WaitPolicy::new(
2599 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2600 || false,
2601 );
2602 let _coordination =
2603 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2604 if let Err(e) = git::snapshot_uncommitted(
2607 &c.worktree,
2608 &self
2609 .state
2610 .dir()
2611 .join("artifacts")
2612 .join(format!("candidate-{label}-recovery")),
2613 )
2614 .await
2615 {
2616 self.state.event(
2617 "preserve",
2618 format!("candidate {label} snapshot incomplete: {e:#}"),
2619 );
2620 }
2621 let rescued = git::rescue_commit(
2622 &c.worktree,
2623 &format!("magi: candidate {label} (uncommitted work)"),
2624 )
2625 .await?;
2626 self.state.note_withheld("implement", &rescued.withheld);
2627 let base = &self.state.base_commit;
2628 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2629 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2630 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2631 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2632 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2633 if c.empty && commits > 0 && !patch.trim().is_empty()
2634 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2635 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2636 || self.state.tally.is_some())
2637 {
2638 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2642 }
2643 let c = &mut self.state.candidates[i];
2644 c.commits = commits;
2645 c.stat = stat;
2646 c.files = files;
2647 let no_changes = commits == 0 || patch.trim().is_empty();
2648 c.empty = no_changes && !interrupted;
2649 if interrupted && no_changes {
2650 c.failed = None;
2651 c.verified_noop = None;
2652 }
2653 if !c.empty {
2654 c.failed = None;
2655 c.verified_noop = None;
2656 }
2657 self.state.event(
2658 "preserve",
2659 format!("candidate {label}: {files} files, {commits} commits"),
2660 );
2661 Ok::<_, anyhow::Error>(())
2662 }
2663 .await;
2664 match result {
2665 Ok(()) => {
2666 self.state.candidate_preservation.remove(&label);
2667 if !interrupted || self.state.candidates[i].files > 0 {
2668 self.state.interrupted_implementations.remove(&label);
2669 }
2670 }
2671 Err(e) => {
2672 let reason = format!("candidate {label} preservation failed: {e:#}");
2673 self.state.event("preserve", &reason);
2674 self.state.candidate_preservation.insert(label, reason);
2675 }
2676 }
2677 if self.state.candidate_preservation.is_empty()
2678 && self.state.status == RunStatus::Blocked
2679 && self.state.candidates.iter().all(|c| c.failed.is_none())
2680 {
2681 self.state.status = RunStatus::Implementing;
2685 self.state.event(
2686 "preserve",
2687 "candidate persistence recovered; implementation can continue",
2688 );
2689 }
2690 self.state.save()?;
2691 }
2692 if !self.state.candidate_preservation.is_empty() {
2693 self.state.status = RunStatus::Blocked;
2694 self.state.save()?;
2695 bail!(
2696 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2697 );
2698 }
2699 Ok(())
2700 }
2701
2702 async fn resume_undelivered(
2730 &mut self,
2731 results: &mut [(usize, SeatState, AgentOutcome)],
2732 sent: &[SeatJob],
2733 prompts: &Prompts,
2734 run_id: &str,
2735 ) {
2736 for (wi, seat, out) in results.iter_mut() {
2737 let Some(dropped) = (match &*out {
2738 AgentOutcome::Dropped(o) => o.dropped.clone(),
2739 _ => None,
2740 }) else {
2741 continue;
2742 };
2743 let Some(job) = sent.get(*wi) else { continue };
2744 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2746 self.state.event(
2747 "implement",
2748 format!(
2749 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2750 work is in the tree",
2751 seat.key, dropped.output_tokens, dropped.why
2752 ),
2753 );
2754 continue;
2755 }
2756 if !has_context(&job.spec, seat, job.sessions) {
2764 self.state.event(
2765 "implement",
2766 format!(
2767 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2768 is no session left to resume",
2769 seat.key, dropped.output_tokens, dropped.why
2770 ),
2771 );
2772 continue;
2773 }
2774 self.state.event(
2775 "implement",
2776 format!(
2777 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2778 conversation",
2779 seat.key, dropped.output_tokens, dropped.why
2780 ),
2781 );
2782 let mut retry = job.clone();
2783 retry.seat = seat.clone();
2784 retry.prompt = prompt::resume_after_drop(&dropped.why);
2785 retry.timeout = retry_budget(job.timeout, true);
2786 retry.stem = format!("{}-resume", job.stem);
2787 let cache = self.state.config.cache_dir();
2788 let ctx = WaveCtx {
2789 carry_seats: false,
2790 pauses: Some((&self.pause, &self.interrupt)),
2791 run: run_id,
2792 node: "implement",
2793 prompts,
2794 cache: cache.as_deref(),
2795 round: None,
2796 };
2797 let (resumed_seat, resumed) =
2798 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2799 *seat = resumed_seat;
2800 *out = resumed;
2801 }
2802 }
2803
2804 async fn resume_seat_handovers(
2869 &mut self,
2870 results: &mut [(usize, SeatState, AgentOutcome)],
2871 sent: &mut [SeatJob],
2872 prompts: &Prompts,
2873 run_id: &str,
2874 ) -> Result<()> {
2875 let instruction = seeded_instruction(&self.state);
2876 let language = self.state.config.graph.language.clone();
2877 let github_language = self.state.config.graph.github_language.clone();
2878 let brief = self
2879 .state
2880 .advice
2881 .as_ref()
2882 .and_then(|a| a.synthesis.as_deref())
2883 .map(str::to_owned);
2884 let attachments = self.state.attachments.clone();
2885 for (wi, seat, out) in results.iter_mut() {
2886 let others: BTreeSet<String> = sent
2889 .iter()
2890 .enumerate()
2891 .filter(|(j, _)| j != wi)
2892 .map(|(_, j)| j.spec.id.clone())
2893 .collect();
2894 let Some(job) = sent.get_mut(*wi) else {
2895 continue;
2896 };
2897 let start = self
2902 .roles
2903 .implementer_roster
2904 .iter()
2905 .position(|s| s.id == job.spec.id)
2906 .unwrap_or(0);
2907 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2908 let mut fallback_attempt = 0usize;
2909 let mut prev: Option<FailClass> = None;
2910 while let Some(cur) = FailClass::of(&*out) {
2911 if !should_hand_over(prev.as_ref(), &cur) {
2912 break;
2913 }
2914 let Some(next) =
2915 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2916 .cloned()
2917 else {
2918 break;
2919 };
2920 tried.insert(next.id.clone());
2921 fallback_attempt += 1;
2922
2923 let coordination = self.coordinate(&job.cwd).await?;
2924 if let Ok(r) = git::rescue_commit(
2925 &job.cwd,
2926 &format!(
2927 "magi: candidate {} (uncommitted work before {} fallback)",
2928 seat.key,
2929 if cur == FailClass::Quota {
2930 "quota"
2931 } else {
2932 "handover"
2933 }
2934 ),
2935 )
2936 .await
2937 {
2938 self.state.note_withheld("implement", &r.withheld);
2939 }
2940 drop(coordination);
2941
2942 record_handover(
2943 &mut self.state,
2944 "implement",
2945 &seat.key,
2946 &seat.agent,
2947 &next.id,
2948 &cur,
2949 &fail_reason(&*out),
2950 );
2951 prev = Some(cur.clone());
2952
2953 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2954 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2955 job.spec = next.clone();
2963 let mut retry = job.clone();
2964 retry.seat = new_seat;
2965 retry.prompt = prompt::implement_in(
2966 &instruction,
2967 &job.cwd.to_string_lossy(),
2968 &language,
2969 &github_language,
2970 brief.as_deref(),
2971 &attachments,
2972 );
2973 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2974 let cache = self.state.config.cache_dir();
2975 let ctx = WaveCtx {
2976 carry_seats: false,
2977 pauses: Some((&self.pause, &self.interrupt)),
2978 run: run_id,
2979 node: "implement",
2980 prompts,
2981 cache: cache.as_deref(),
2982 round: None,
2983 };
2984 let (fallback_seat, fallback_out) = run_one(
2985 retry,
2986 Arc::clone(&self.sem),
2987 &ctx,
2988 &mut self.state,
2989 fallback_attempt,
2990 )
2991 .await;
2992 *seat = fallback_seat;
2993 *out = fallback_out;
2994 }
2995 }
2996 Ok(())
2997 }
2998
2999 async fn resume_unconfirmed_commands(
3023 &mut self,
3024 results: &mut [(usize, SeatState, AgentOutcome)],
3025 sent: &[SeatJob],
3026 prompts: &Prompts,
3027 run_id: &str,
3028 ) {
3029 for (wi, seat, out) in results.iter_mut() {
3030 let AgentOutcome::Ok(o) = &*out else {
3031 continue;
3032 };
3033 if !has_unconfirmed_command(&o.commands) {
3034 continue;
3035 }
3036 let Some(job) = sent.get(*wi) else { continue };
3037 if !has_context(&job.spec, seat, job.sessions) {
3038 self.state.event(
3039 "implement",
3040 format!(
3041 "{}: the reply named a command whose own CLI never confirmed the exit \
3042 status of, but there is no session left to resume",
3043 seat.key
3044 ),
3045 );
3046 continue;
3047 }
3048 self.state.event(
3049 "implement",
3050 format!(
3051 "{}: the reply named a command whose own CLI never confirmed the exit \
3052 status of; resuming the conversation",
3053 seat.key
3054 ),
3055 );
3056 let mut retry = job.clone();
3057 retry.seat = seat.clone();
3058 retry.prompt = prompt::resume_incomplete(
3059 "a command in your last reply had no confirmed exit status",
3060 );
3061 retry.timeout = retry_budget(job.timeout, true);
3062 retry.stem = format!("{}-confirm", job.stem);
3063 let cache = self.state.config.cache_dir();
3064 let ctx = WaveCtx {
3065 carry_seats: false,
3066 pauses: Some((&self.pause, &self.interrupt)),
3067 run: run_id,
3068 node: "implement",
3069 prompts,
3070 cache: cache.as_deref(),
3071 round: None,
3072 };
3073 let (resumed_seat, resumed) =
3074 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3075 *seat = resumed_seat;
3076 *out = resumed;
3077 }
3078 }
3079
3080 async fn continue_fix_report(
3101 &mut self,
3102 mut seat: SeatState,
3103 parse_err: String,
3104 job: &SeatJob,
3105 prompts: &Prompts,
3106 run_id: &str,
3107 round: usize,
3108 ) -> (
3109 SeatState,
3110 Option<FixReport>,
3111 Option<String>,
3112 ContinuationRecord,
3113 ) {
3114 let mut last_err = parse_err;
3115 let mut cumulative_wait_ms = 0u64;
3116 let mut attempts = 0usize;
3117 loop {
3118 if !has_context(&job.spec, &seat, job.sessions) {
3119 self.state.event(
3120 "fix",
3121 format!(
3122 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3123 session left to resume into"
3124 ),
3125 );
3126 let outcome = if attempts == 0 {
3127 ContinuationOutcome::NoSession
3128 } else {
3129 ContinuationOutcome::Exhausted
3130 };
3131 return (
3132 seat,
3133 None,
3134 Some(format!("unparsable fix report: {last_err}")),
3135 ContinuationRecord {
3136 attempts,
3137 cumulative_wait_ms,
3138 outcome,
3139 },
3140 );
3141 }
3142 if attempts >= MAX_FIX_CONTINUATIONS {
3143 self.state.event(
3144 "fix",
3145 format!(
3146 "round {round}: fixer's reply still had no adoption report after \
3147 {attempts} continuation(s) ({last_err}); giving up"
3148 ),
3149 );
3150 return (
3151 seat,
3152 None,
3153 Some(format!(
3154 "unparsable fix report after {attempts} continuation(s): {last_err}"
3155 )),
3156 ContinuationRecord {
3157 attempts,
3158 cumulative_wait_ms,
3159 outcome: ContinuationOutcome::Exhausted,
3160 },
3161 );
3162 }
3163 attempts += 1;
3164 self.state.event(
3165 "fix",
3166 format!(
3167 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3168 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3169 ),
3170 );
3171 let mut retry = job.clone();
3172 retry.seat = seat.clone();
3173 retry.prompt = prompt::resume_incomplete(&last_err);
3174 retry.timeout = retry_budget(job.timeout, true);
3175 retry.stem = format!("{}-continue{attempts}", job.stem);
3176 let cache = self.state.config.cache_dir();
3177 let ctx = WaveCtx {
3178 carry_seats: false,
3179 pauses: Some((&self.pause, &self.interrupt)),
3180 run: run_id,
3181 node: "fix",
3182 prompts,
3183 cache: cache.as_deref(),
3184 round: Some(round),
3185 };
3186 let (resumed_seat, resumed_out) = run_one(
3187 retry,
3188 Arc::clone(&self.sem),
3189 &ctx,
3190 &mut self.state,
3191 attempts,
3192 )
3193 .await;
3194 seat = resumed_seat;
3195 match resumed_out {
3196 AgentOutcome::Ok(o) => {
3197 cumulative_wait_ms += o.duration_ms;
3198 match verdict::extract_json::<FixReport>(&o.text) {
3199 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3200 self.state.event(
3201 "fix",
3202 format!(
3203 "round {round}: fixer's adoption report recovered after \
3204 {attempts} continuation(s)"
3205 ),
3206 );
3207 return (
3208 seat,
3209 Some(report),
3210 None,
3211 ContinuationRecord {
3212 attempts,
3213 cumulative_wait_ms,
3214 outcome: ContinuationOutcome::Resumed,
3215 },
3216 );
3217 }
3218 Ok(_) => {
3226 last_err = "the reply parsed, but it reported a command whose own CLI \
3227 never confirmed an exit status"
3228 .to_owned();
3229 }
3230 Err(e) => last_err = e.to_string(),
3231 }
3232 }
3233 AgentOutcome::Quota(o) => {
3234 cumulative_wait_ms += o.duration_ms;
3235 self.state.quota.push(QuotaLoss {
3236 seat: seat.key.clone(),
3237 node: "fix".to_owned(),
3238 at: Timestamp::now(),
3239 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3240 });
3241 self.state.event(
3242 "fix",
3243 format!(
3244 "round {round}: continuation rate limited (quota); not retrying now"
3245 ),
3246 );
3247 return (
3248 seat,
3249 None,
3250 Some("rate limited (quota) while recovering the fix report".to_owned()),
3251 ContinuationRecord {
3252 attempts,
3253 cumulative_wait_ms,
3254 outcome: ContinuationOutcome::QuotaLost,
3255 },
3256 );
3257 }
3258 AgentOutcome::Dropped(o) => {
3259 cumulative_wait_ms += o.duration_ms;
3260 let why = o
3261 .dropped
3262 .as_ref()
3263 .map(|d| d.why.as_str())
3264 .unwrap_or("the CLI ended the stream without delivering its answer");
3265 last_err = format!("the CLI dropped the stream ({why})");
3266 }
3267 AgentOutcome::Failed(e) => last_err = e,
3268 AgentOutcome::ResourceBlocked(e) => {
3271 return (
3272 seat,
3273 None,
3274 Some(format!("the continuation was not started: {e}")),
3275 ContinuationRecord {
3276 attempts: attempts.saturating_sub(1),
3277 cumulative_wait_ms,
3278 outcome: ContinuationOutcome::NoSession,
3279 },
3280 );
3281 }
3282 }
3283 }
3284 }
3285
3286 fn after_implement(&mut self) -> Result<()> {
3287 if !self.state.candidate_preservation.is_empty() {
3288 bail!("candidate preservation incomplete; judging is unsafe");
3289 }
3290 if self.state.leaks.is_empty() {
3292 let cfg = self.state.config.blind.clone();
3293 let mut leaks = Vec::new();
3294 for c in &self.state.candidates {
3295 let Some(patch) =
3296 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3297 else {
3298 continue;
3299 };
3300 leaks.extend(blind::scan(
3301 &format!("candidate {} patch", c.label),
3302 &patch,
3303 &cfg.vendor_tokens,
3304 ));
3305 }
3306 if !leaks.is_empty() {
3307 let summary = leaks
3308 .iter()
3309 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3310 .collect::<Vec<_>>()
3311 .join(", ");
3312 match cfg.on_leak {
3313 LeakPolicy::Fail => {
3314 self.state.status = RunStatus::Failed;
3315 self.state
3316 .event("blind", format!("vendor text in a patch: {summary}"));
3317 self.state.leaks = leaks;
3318 self.state.save()?;
3319 self.settle_questions();
3320 bail!(
3321 "blind.on_leak = \"fail\" and vendor text reached a \
3322 judged patch: {summary}"
3323 );
3324 }
3325 LeakPolicy::Redact => self.state.event(
3326 "blind",
3327 format!("redacting vendor text for judging: {summary}"),
3328 ),
3329 LeakPolicy::Warn => self.state.event(
3330 "blind",
3331 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3332 ),
3333 }
3334 self.state.leaks = leaks;
3335 }
3336 }
3337
3338 if self.state.viable().is_empty() {
3339 if self.state.all_candidates_verified_noop() {
3340 self.state.status = RunStatus::VerifiedNoop;
3351 self.state.save()?;
3352 self.settle_questions();
3353 return Ok(());
3354 }
3355 self.state.status = RunStatus::Failed;
3356 self.state.save()?;
3357 self.settle_questions();
3358 bail!("no candidate produced a change; nothing to judge");
3359 }
3360 self.state.status = RunStatus::Judging;
3361 self.state.save()?;
3362 Ok(())
3363 }
3364
3365 async fn judge(&mut self) -> Result<()> {
3368 let run_id = self.state.id.clone();
3373 let prompts = self.state.config.prompts.clone();
3374 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3375 return Ok(());
3376 }
3377 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3378 if viable.len() == 1 {
3379 self.state.judge_skipped = true;
3386 self.state.event(
3387 "judge",
3388 format!(
3389 "only candidate {} produced a change; judging skipped",
3390 viable[0].label
3391 ),
3392 );
3393 self.state.save()?;
3394 return Ok(());
3395 }
3396 self.state.status = RunStatus::Judging;
3397
3398 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3399 let language = self.state.config.graph.language.clone();
3400 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3401 let sessions = self.state.config.graph.sessions;
3402 let artifacts = agent::artifacts_dir(&self.state.dir());
3403 let root = self.state.worktree_root();
3404 let base_short = short(&self.state.base_commit);
3405
3406 let mut jobs = Vec::new();
3407 let mut orders = Vec::new();
3408 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3409 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3410 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3411 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3412 let seat_key = format!("judge-{}", j + 1);
3413 let seat = self.seat(&seat_key, &spec.id);
3414 jobs.push(SeatJob {
3415 prompt: prompt::judge(
3416 &self.state.instruction,
3417 &views,
3418 self.roles.judges.len(),
3419 &base_short,
3420 &language,
3421 ),
3422 spec,
3423 seat,
3424 cwd: root.join(format!("judge-{}", j + 1)),
3425 timeout,
3426 allow_write: false,
3427 sessions,
3428 artifacts: artifacts.clone(),
3429 stem: format!("judge-{}", j + 1),
3430 handover: None,
3431 });
3432 }
3433
3434 self.state.event(
3435 "judge",
3436 format!(
3437 "{} judges ranking {} candidates blind",
3438 jobs.len(),
3439 viable.len()
3440 ),
3441 );
3442 let labels_for_check = labels.clone();
3443 let mut quota_losses = Vec::new();
3444 let cache = self.state.config.cache_dir();
3445 let ctx = WaveCtx {
3446 carry_seats: false,
3447 pauses: Some((&self.pause, &self.interrupt)),
3448 run: &run_id,
3449 node: "judge",
3450 prompts: &prompts,
3451 cache: cache.as_deref(),
3452 round: None,
3453 };
3454 let results = ask_json_wave::<Ranking>(
3455 jobs,
3456 Arc::clone(&self.sem),
3457 self.state.config.graph.retries,
3458 &self.roles.judge_roster,
3459 &ctx,
3460 &mut quota_losses,
3461 &mut self.state,
3462 &move |r: &Ranking| r.validate(&labels_for_check),
3463 )
3464 .await;
3465 if self.state.resource_block.is_some() {
3466 return self.stop_for_resource();
3467 }
3468 self.state.quota.extend(quota_losses);
3469
3470 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3471 let agent_id = seat.agent.clone();
3472 self.state.seats.insert(seat.key.clone(), seat);
3473 let mut record = Judgement {
3474 judge: j + 1,
3475 seat: format!("judge-{}", j + 1),
3476 agent: agent_id,
3477 ranking: Vec::new(),
3478 reasons: BTreeMap::new(),
3479 confidence: None,
3480 strengths: BTreeMap::new(),
3481 order: orders[j].clone(),
3482 failed: None,
3483 duration_ms: 0,
3484 };
3485 match res {
3486 Ok((ranking, out)) => {
3487 record.ranking = ranking.normalized();
3488 record.reasons = ranking.reasons;
3489 record.strengths = ranking.strengths;
3490 record.confidence = ranking.confidence;
3491 record.duration_ms = out.duration_ms;
3492 self.state.event(
3493 "judge",
3494 format!(
3495 "judge {} ranked {}",
3496 j + 1,
3497 record.ranking.iter().collect::<String>()
3498 ),
3499 );
3500 }
3501 Err(e) => {
3502 record.failed = Some(e.to_string());
3503 self.state
3504 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3505 }
3506 }
3507 self.state.judgements.push(record);
3508 self.state.save()?;
3509 }
3510 Ok(())
3511 }
3512
3513 async fn deliberate(&mut self) -> Result<()> {
3516 let run_id = self.state.id.clone();
3521 let prompts = self.state.config.prompts.clone();
3522 if !self.state.deliberation.is_empty() {
3523 return Ok(());
3524 }
3525 let tops: Vec<char> = self
3526 .state
3527 .judgements
3528 .iter()
3529 .filter_map(|j| j.ranking.first().copied())
3530 .collect();
3531 let rounds = self.state.config.graph.deliberate_rounds;
3532 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3533 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3534 self.state.event(
3535 "deliberate",
3536 format!("judges agreed on {} outright; no deliberation", tops[0]),
3537 );
3538 }
3539 self.state.status = RunStatus::Voting;
3540 self.state.save()?;
3541 return Ok(());
3542 }
3543
3544 self.state.status = RunStatus::Deliberating;
3545 self.state.event(
3546 "deliberate",
3547 format!(
3548 "split: first choices were {} — opening {rounds} round(s)",
3549 tops.iter().collect::<String>()
3550 ),
3551 );
3552
3553 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3554 let language = self.state.config.graph.language.clone();
3555 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3556 let sessions = self.state.config.graph.sessions;
3557 let artifacts = agent::artifacts_dir(&self.state.dir());
3558 let root = self.state.worktree_root();
3559 let base_short = short(&self.state.base_commit);
3560
3561 for round in 1..=rounds {
3565 let mut turns: Vec<DeliberationTurn> = Vec::new();
3566 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3567 if self.state.judgements[j].failed.is_some() {
3568 continue;
3569 }
3570 let seat_key = format!("judge-{}", j + 1);
3571 let spec = self.occupant(&seat_key, spec);
3572 let mut seat = self.seat(&seat_key, &spec.id);
3573 let transcript = self.transcript(&turns, j);
3574 let build = |context: Option<&str>| {
3575 prompt::deliberate(
3576 &self.state.instruction,
3577 context,
3578 &transcript,
3579 round,
3580 rounds,
3581 &language,
3582 )
3583 };
3584 let block = self.candidate_block(&viable, &base_short);
3585 let full = build(Some(&block));
3586 let text = if has_context(&spec, &seat, sessions) {
3587 build(None)
3588 } else {
3589 full.clone()
3590 };
3591 let job = SeatJob {
3592 spec,
3593 seat: seat.clone(),
3594 prompt: text,
3595 cwd: root.join(format!("judge-{}", j + 1)),
3596 timeout,
3597 allow_write: false,
3598 sessions,
3599 artifacts: artifacts.clone(),
3600 stem: format!("delib-{round}-judge-{}", j + 1),
3601 handover: Some(full),
3602 };
3603 let cache = self.state.config.cache_dir();
3604 let ctx = WaveCtx {
3605 carry_seats: false,
3606 pauses: Some((&self.pause, &self.interrupt)),
3607 run: &run_id,
3608 node: "deliberate",
3609 prompts: &prompts,
3610 cache: cache.as_deref(),
3611 round: None,
3612 };
3613 let mut losses = Vec::new();
3617 let mut results = ask_wave_with::<String>(
3618 vec![job],
3619 Arc::clone(&self.sem),
3620 0,
3621 &self.roles.judge_roster,
3622 &ctx,
3623 &mut losses,
3624 &mut self.state,
3625 &|text: &str| {
3626 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3627 },
3628 )
3629 .await;
3630 self.state.quota.extend(losses);
3631 let (updated, res, _) = results.pop().expect("one job in, one result out");
3632 seat = updated;
3633 let agent_id = seat.agent.clone();
3634 self.state.seats.insert(seat.key.clone(), seat);
3635 let body = match res {
3636 Ok((body, _)) => body,
3637 Err(e) => {
3640 self.state
3641 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3642 continue;
3643 }
3644 };
3645 let tentative = verdict::extract_json::<Position>(&body)
3646 .ok()
3647 .and_then(|p| p.tentative)
3648 .and_then(|s| s.trim().chars().next())
3649 .map(|c| c.to_ascii_uppercase());
3650 self.state.event(
3651 "deliberate",
3652 format!(
3653 "round {round}: judge {} now favours {}",
3654 j + 1,
3655 tentative.map_or("—".to_owned(), |c| c.to_string())
3656 ),
3657 );
3658 turns.push(DeliberationTurn {
3659 judge: j + 1,
3660 agent: agent_id,
3661 body: blind::sanitize_prose(&body, &self.state.config.blind),
3662 tentative,
3663 });
3664 }
3665 self.state
3666 .deliberation
3667 .push(DeliberationRound { round, turns });
3668 self.state.save()?;
3669 }
3670
3671 self.state.status = RunStatus::Voting;
3672 self.state.save()?;
3673 Ok(())
3674 }
3675
3676 async fn vote(&mut self) -> Result<()> {
3679 let run_id = self.state.id.clone();
3684 let prompts = self.state.config.prompts.clone();
3685 if !self.state.votes.is_empty() {
3686 return Ok(());
3687 }
3688 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3689 if viable.len() == 1 {
3690 return Ok(());
3691 }
3692 self.state.status = RunStatus::Voting;
3693
3694 let language = self.state.config.graph.language.clone();
3695 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3696 let sessions = self.state.config.graph.sessions;
3697 let artifacts = agent::artifacts_dir(&self.state.dir());
3698 let root = self.state.worktree_root();
3699 let base_short = short(&self.state.base_commit);
3700 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3701
3702 let mut jobs = Vec::new();
3703 let mut seats_at = Vec::new();
3704 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3705 if self
3706 .state
3707 .judgements
3708 .get(j)
3709 .is_some_and(|r| r.failed.is_some())
3710 {
3711 continue;
3712 }
3713 let seat_key = format!("judge-{}", j + 1);
3714 let spec = self.occupant(&seat_key, spec);
3715 let seat = self.seat(&seat_key, &spec.id);
3716 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3717 let text = if has_context(&spec, &seat, sessions) {
3718 prompt::final_vote(&viable, &language)
3719 } else {
3720 full.clone()
3721 };
3722 jobs.push(SeatJob {
3723 spec,
3724 seat,
3725 prompt: text,
3726 cwd: root.join(format!("judge-{}", j + 1)),
3727 timeout,
3728 allow_write: false,
3729 sessions,
3730 artifacts: artifacts.clone(),
3731 stem: format!("vote-judge-{}", j + 1),
3732 handover: Some(full),
3733 });
3734 seats_at.push(j);
3735 }
3736
3737 self.state.event(
3738 "vote",
3739 format!(
3740 "collecting {} final votes one by one, privately",
3741 jobs.len()
3742 ),
3743 );
3744 let allowed = viable.clone();
3745 let mut quota_losses = Vec::new();
3746 let cache = self.state.config.cache_dir();
3747 let ctx = WaveCtx {
3748 carry_seats: false,
3749 pauses: Some((&self.pause, &self.interrupt)),
3750 run: &run_id,
3751 node: "vote",
3752 prompts: &prompts,
3753 cache: cache.as_deref(),
3754 round: None,
3755 };
3756 let results = ask_json_wave::<FinalVote>(
3757 jobs,
3758 Arc::clone(&self.sem),
3759 self.state.config.graph.retries,
3760 &self.roles.judge_roster,
3761 &ctx,
3762 &mut quota_losses,
3763 &mut self.state,
3764 &move |v: &FinalVote| match v.label() {
3765 Some(c) if allowed.contains(&c) => Ok(()),
3766 other => bail!("vote {other:?} is not one of {allowed:?}"),
3767 },
3768 )
3769 .await;
3770 if self.state.resource_block.is_some() {
3771 return self.stop_for_resource();
3772 }
3773 self.state.quota.extend(quota_losses);
3774
3775 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3776 let agent_id = seat.agent.clone();
3777 self.state.seats.insert(seat.key.clone(), seat);
3778 let initial = self
3779 .state
3780 .judgements
3781 .get(j)
3782 .and_then(|r| r.ranking.first().copied());
3783 let mut record = VoteRecord {
3784 judge: j + 1,
3785 agent: agent_id,
3786 vote: None,
3787 reason: String::new(),
3788 changed: false,
3789 };
3790 match res {
3791 Ok((v, _)) => {
3792 record.vote = v.label();
3793 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3794 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3795 self.state.event(
3796 "vote",
3797 format!(
3798 "judge {} voted {}{}",
3799 j + 1,
3800 record.vote.unwrap_or('?'),
3801 if record.changed { " (changed)" } else { "" }
3802 ),
3803 );
3804 }
3805 Err(e) => {
3806 self.state
3807 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3808 }
3809 }
3810 self.state.votes.push(record);
3811 self.state.save()?;
3812 }
3813 Ok(())
3814 }
3815
3816 fn tally(&mut self) -> Result<()> {
3819 if self.state.tally.is_some() {
3820 return Ok(());
3821 }
3822 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3823 let tops: Vec<char> = self
3824 .state
3825 .judgements
3826 .iter()
3827 .filter_map(|j| j.ranking.first().copied())
3828 .collect();
3829 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3830
3831 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3834 let mut cast: Vec<char> = Vec::new();
3835 for (i, j) in self.state.judgements.iter().enumerate() {
3836 let vote = self
3837 .state
3838 .votes
3839 .iter()
3840 .find(|v| v.judge == i + 1)
3841 .and_then(|v| v.vote)
3842 .or_else(|| j.ranking.first().copied());
3843 if let Some(v) = vote {
3844 *first_choice.entry(v).or_insert(0) += 1;
3845 cast.push(v);
3846 }
3847 }
3848
3849 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3850 for j in &self.state.judgements {
3851 let n = j.ranking.len();
3852 for (pos, label) in j.ranking.iter().enumerate() {
3853 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3854 }
3855 }
3856
3857 let best = first_choice.values().copied().max().unwrap_or(0);
3858 let mut leaders: Vec<char> = first_choice
3859 .iter()
3860 .filter(|(_, v)| **v == best)
3861 .map(|(k, _)| *k)
3862 .collect();
3863 let mut tie_break = None;
3864 if leaders.len() > 1 {
3865 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3866 let borda_leaders: Vec<char> = leaders
3867 .iter()
3868 .copied()
3869 .filter(|l| borda[l] == top_borda)
3870 .collect();
3871 tie_break = Some(if borda_leaders.len() == 1 {
3872 format!(
3873 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3874 leaders.len()
3875 )
3876 } else {
3877 format!(
3878 "{} way tie on both first-choice votes and Borda points, broken by label order",
3879 leaders.len()
3880 )
3881 });
3882 leaders = borda_leaders;
3883 leaders.sort_unstable();
3884 }
3885 let winner = *leaders
3886 .first()
3887 .or(viable.first())
3888 .context("no candidate to declare a winner from")?;
3889
3890 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3891 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3892 let deliberated = !self.state.deliberation.is_empty();
3893
3894 let quota_seats: std::collections::BTreeSet<&str> =
3898 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3899 let mut present = 0usize;
3900 for (i, j) in self.state.judgements.iter().enumerate() {
3901 if quota_seats.contains(j.seat.as_str()) {
3902 continue;
3903 }
3904 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3905 let voted = self
3906 .state
3907 .votes
3908 .iter()
3909 .any(|v| v.judge == i + 1 && v.vote.is_some());
3910 if ranked || voted {
3911 present += 1;
3912 }
3913 }
3914 let needs_quorum = viable.len() > 1;
3920 let judges_total = if needs_quorum {
3921 self.roles.judges.len()
3922 } else {
3923 0
3924 };
3925 let quorum = if needs_quorum {
3926 judges_total / 2 + 1
3927 } else {
3928 0
3929 };
3930 let met_quorum = !needs_quorum || present >= quorum;
3931 let uncontested = (!needs_quorum).then(|| {
3932 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3933 });
3934
3935 self.state.event(
3936 "tally",
3937 match &uncontested {
3938 Some(reason) => format!("winner {winner} — {reason}"),
3939 None => format!(
3940 "winner {winner} — votes {} | initial {} | {} changed | \
3941 {present}/{judges_total} judges{}",
3942 first_choice
3943 .iter()
3944 .map(|(k, v)| format!("{k}:{v}"))
3945 .collect::<Vec<_>>()
3946 .join(" "),
3947 if unanimous_initial {
3948 "unanimous"
3949 } else {
3950 "split"
3951 },
3952 changed_votes,
3953 if met_quorum {
3954 String::new()
3955 } else {
3956 format!(" — below quorum ({quorum} required)")
3957 },
3958 ),
3959 },
3960 );
3961 if !met_quorum {
3962 self.state.event(
3963 "stall",
3964 format!(
3965 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3966 the run stops here, resumable"
3967 ),
3968 );
3969 }
3970 self.state.tally = Some(Tally {
3971 first_choice,
3972 borda,
3973 winner,
3974 rankings: tops.len(),
3975 unanimous_initial,
3976 deliberated,
3977 changed_votes,
3978 unanimous_final,
3979 tie_break,
3980 judges: judges_total,
3981 present,
3982 quorum,
3983 met_quorum,
3984 uncontested,
3985 });
3986 self.state.status = if met_quorum {
3987 RunStatus::Reviewing
3988 } else {
3989 RunStatus::Stalled
3990 };
3991 self.state.save()?;
3992 Ok(())
3993 }
3994
3995 #[allow(clippy::too_many_lines)]
4016 async fn recover_stall(&mut self) -> Result<bool> {
4017 let run_id = self.state.id.clone();
4022 let prompts = self.state.config.prompts.clone();
4023 let quota_seats: BTreeSet<&str> =
4028 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4029 let absent: Vec<String> = self
4030 .state
4031 .judgements
4032 .iter()
4033 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4034 .map(|j| j.seat.clone())
4035 .collect();
4036 if absent.is_empty() {
4037 return Ok(false);
4038 }
4039 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4040 if viable.len() <= 1 {
4041 return Ok(false);
4042 }
4043 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4044 let language = self.state.config.graph.language.clone();
4045 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4046 let sessions = self.state.config.graph.sessions;
4047 let artifacts = agent::artifacts_dir(&self.state.dir());
4048 let root = self.state.worktree_root();
4049 let base_short = short(&self.state.base_commit);
4050 let candidates: Vec<Candidate> = viable.clone();
4051
4052 let mut positions: Vec<usize> = absent
4054 .iter()
4055 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4056 .collect();
4057 if positions.is_empty() {
4058 return Ok(false);
4059 }
4060 positions.sort_unstable();
4061 positions.dedup();
4062
4063 let mut judge_jobs = Vec::new();
4065 for &j in &positions {
4066 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4067 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4068 let seat_key = format!("judge-{}", j + 1);
4069 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4070 let seat = self.seat(&seat_key, &spec.id);
4071 judge_jobs.push(SeatJob {
4072 spec,
4073 seat,
4074 prompt: prompt::judge(
4075 &self.state.instruction,
4076 &views,
4077 self.roles.judges.len(),
4078 &base_short,
4079 &language,
4080 ),
4081 cwd: root.join(seat_key),
4082 timeout,
4083 allow_write: false,
4084 sessions,
4085 artifacts: artifacts.clone(),
4086 stem: format!("judge-{}-recover", j + 1),
4087 handover: None,
4088 });
4089 }
4090
4091 let labels_for_check = labels.clone();
4092 let mut judge_losses = Vec::new();
4093 let retries = self.state.config.graph.retries;
4094 let cache = self.state.config.cache_dir();
4095 let ctx = WaveCtx {
4096 carry_seats: false,
4097 pauses: Some((&self.pause, &self.interrupt)),
4098 run: &run_id,
4099 node: "judge",
4100 prompts: &prompts,
4101 cache: cache.as_deref(),
4102 round: None,
4103 };
4104 let results = ask_json_wave::<Ranking>(
4105 judge_jobs,
4106 Arc::clone(&self.sem),
4107 retries,
4108 &self.roles.judge_roster,
4109 &ctx,
4110 &mut judge_losses,
4111 &mut self.state,
4112 &move |r: &Ranking| r.validate(&labels_for_check),
4113 )
4114 .await;
4115 if self.state.resource_block.is_some() {
4116 self.stop_for_resource()?;
4117 return Ok(false);
4118 }
4119
4120 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4122 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4123 let agent_id = seat.agent.clone();
4124 self.state.seats.insert(seat.key.clone(), seat);
4125 let record = &mut self.state.judgements[j];
4126 match res {
4127 Ok((ranking, out)) => {
4128 record.agent = agent_id;
4129 record.ranking = ranking.normalized();
4130 record.reasons = ranking.reasons;
4131 record.strengths = ranking.strengths;
4132 record.confidence = ranking.confidence;
4133 record.failed = None;
4134 record.duration_ms = out.duration_ms;
4135 recovered.insert(j);
4136 self.state.event(
4137 "recover",
4138 format!("judge {} ranked again after the limit", j + 1),
4139 );
4140 }
4141 Err(e) => {
4142 self.state
4143 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4144 }
4145 }
4146 }
4147
4148 let mut vote_jobs = Vec::new();
4150 let mut vote_pos: Vec<usize> = Vec::new();
4151 for &j in &recovered {
4152 let seat_key = format!("judge-{}", j + 1);
4153 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4154 let seat = self.seat(&seat_key, &spec.id);
4155 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4156 let text = if has_context(&spec, &seat, sessions) {
4157 prompt::final_vote(&labels, &language)
4158 } else {
4159 full.clone()
4160 };
4161 vote_jobs.push(SeatJob {
4162 spec,
4163 seat,
4164 prompt: text,
4165 cwd: root.join(seat_key),
4166 timeout,
4167 allow_write: false,
4168 sessions,
4169 artifacts: artifacts.clone(),
4170 stem: format!("vote-judge-{}-recover", j + 1),
4171 handover: Some(full),
4172 });
4173 vote_pos.push(j);
4174 }
4175 let allowed = labels.clone();
4176 let mut vote_losses = Vec::new();
4177 let vote_retries = self.state.config.graph.retries;
4178 let vote_cache = self.state.config.cache_dir();
4179 let ctx = WaveCtx {
4180 carry_seats: false,
4181 pauses: Some((&self.pause, &self.interrupt)),
4182 run: &run_id,
4183 node: "vote",
4184 prompts: &prompts,
4185 cache: vote_cache.as_deref(),
4186 round: None,
4187 };
4188 let votes = ask_json_wave::<FinalVote>(
4189 vote_jobs,
4190 Arc::clone(&self.sem),
4191 vote_retries,
4192 &self.roles.judge_roster,
4193 &ctx,
4194 &mut vote_losses,
4195 &mut self.state,
4196 &move |v: &FinalVote| match v.label() {
4197 Some(c) if allowed.contains(&c) => Ok(()),
4198 other => bail!("vote {other:?} is not one of {allowed:?}"),
4199 },
4200 )
4201 .await;
4202 if self.state.resource_block.is_some() {
4203 self.stop_for_resource()?;
4204 return Ok(false);
4205 }
4206 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4207 let agent_id = seat.agent.clone();
4208 self.state.seats.insert(seat.key.clone(), seat);
4209 match res {
4210 Ok((v, _)) => {
4211 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4212 rec.vote = v.label();
4213 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4214 } else {
4215 self.state.votes.push(VoteRecord {
4216 judge: j + 1,
4217 agent: agent_id,
4218 vote: v.label(),
4219 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4220 changed: false,
4221 });
4222 }
4223 self.state.event(
4224 "recover",
4225 format!("judge {} voted again after the limit", j + 1),
4226 );
4227 }
4228 Err(e) => {
4229 self.state
4230 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4231 }
4232 }
4233 }
4234
4235 let recovered_keys: BTreeSet<String> = recovered
4239 .iter()
4240 .map(|&j| format!("judge-{}", j + 1))
4241 .collect();
4242 self.state
4243 .quota
4244 .retain(|q| !recovered_keys.contains(&q.seat));
4245 for loss in judge_losses.into_iter().chain(vote_losses) {
4249 if recovered_keys.contains(&loss.seat) {
4250 continue;
4251 }
4252 self.state.quota.retain(|q| q.seat != loss.seat);
4253 self.state.quota.push(loss);
4254 }
4255
4256 self.state.tally = None;
4258 self.tally()?;
4259 Ok(self
4260 .state
4261 .tally
4262 .as_ref()
4263 .map(|t| t.met_quorum)
4264 .unwrap_or(false))
4265 }
4266
4267 async fn graft(&mut self) -> Result<()> {
4283 let Err(e) = self.graft_inner().await else {
4284 return Ok(());
4285 };
4286 if e.is::<crate::repo_lock::ResourceBlocked>() {
4287 return Err(e);
4288 }
4289 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4290 self.state.status = RunStatus::Blocked;
4291 self.state.event("graft", format!("blocked: {e:#}"));
4292 self.state.save()?;
4293 return Err(e);
4294 }
4295 if let Some(rec) = self.state.graft.clone()
4300 && !rec.outcome.settled()
4301 {
4302 if let Some(winner) = self.state.winner().cloned() {
4303 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4304 if re.is::<crate::repo_lock::ResourceBlocked>() {
4305 return Err(re);
4306 }
4307 self.state.status = RunStatus::Blocked;
4308 self.state.event("graft", format!("blocked: {re:#}"));
4309 self.state.save()?;
4310 return Err(re);
4311 }
4312 }
4313 if let Some(g) = self.state.graft.as_mut() {
4314 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4315 }
4316 }
4317 self.state.event("graft", format!("skipped: {e:#}"));
4318 if let Err(e) = self.state.save() {
4319 tracing::warn!("save after graft: {e:#}");
4320 }
4321 Ok(())
4322 }
4323
4324 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4327 let _coordination = self.coordinate(wt).await?;
4328 crate::recovery::restore_repair(wt, pre)
4329 .await
4330 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4331 }
4332
4333 fn graft_skipped(&mut self, why: &str) {
4335 let message = format!("skipped: {why}");
4336 if !self
4337 .state
4338 .events
4339 .iter()
4340 .any(|e| e.node == "graft" && e.message == message)
4341 {
4342 self.state.event("graft", message);
4343 }
4344 }
4345
4346 async fn graft_inner(&mut self) -> Result<()> {
4347 if let Some(rec) = self.state.graft.clone()
4350 && !rec.outcome.settled()
4351 {
4352 if self.state.reviews.is_empty()
4356 && let Some(winner) = self.state.winner().cloned()
4357 {
4358 self.graft_restore(&winner.worktree, &rec.pre).await?;
4359 }
4360 if rec.outcome == GraftOutcome::WaitingForRestore {
4361 self.state.graft = None;
4362 self.state.save()?;
4363 } else {
4364 if let Some(g) = self.state.graft.as_mut() {
4365 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4366 }
4367 self.state.event(
4368 "graft",
4369 "rolled back: the previous round was interrupted; the winner is untouched",
4370 );
4371 self.state.save()?;
4372 return Ok(());
4373 }
4374 }
4375 let plan = match graft::plan(&self.state) {
4376 Ok(plan) => plan,
4377 Err(skip) => {
4378 if skip != graft::Skip::AlreadyDone {
4379 self.graft_skipped(skip.reason());
4380 self.state.save()?;
4381 }
4382 return Ok(());
4383 }
4384 };
4385 let Some(winner) = self.state.winner().cloned() else {
4386 return Ok(());
4387 };
4388 let wt = winner.worktree.clone();
4389 if !git::is_clean(&wt).await? {
4390 self.graft_skipped("the winner's worktree has uncommitted changes");
4391 self.state.save()?;
4392 return Ok(());
4393 }
4394 let pre = git::rev_parse(&wt, "HEAD").await?;
4395 self.state.graft = Some(GraftRecord {
4398 pre: pre.clone(),
4399 post: None,
4400 lines: None,
4401 limit: None,
4402 agent: None,
4403 outcome: GraftOutcome::Running,
4404 requested: graft::requested(&plan),
4405 adoption: None,
4406 adoption_error: None,
4407 });
4408 self.state.event(
4409 "graft",
4410 format!(
4411 "porting strengths of candidate(s) {} onto {}",
4412 plan.strengths
4413 .iter()
4414 .map(|(l, _, _)| l.to_string())
4415 .collect::<Vec<_>>()
4416 .join(", "),
4417 plan.winner
4418 ),
4419 );
4420 self.state.save()?;
4421
4422 let fix_prompt = prompt::graft(
4423 &self.state.instruction,
4424 plan.winner,
4425 &plan.strengths,
4426 &self.state.config.graph.language,
4427 );
4428 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4429 let sessions = self.state.config.graph.sessions;
4430 let artifacts = agent::artifacts_dir(&self.state.dir());
4431 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4432 let (_, seat, out) = self
4433 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4434 SeatJob {
4435 prompt: fix_prompt.clone(),
4436 spec,
4437 seat,
4438 cwd: wt.clone(),
4439 timeout,
4440 allow_write: true,
4441 sessions,
4442 artifacts: artifacts.clone(),
4443 stem: "graft".to_owned(),
4444 handover: None,
4445 }
4446 })
4447 .await;
4448 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4449 self.state.graft.as_mut().expect("graft record").outcome =
4452 GraftOutcome::WaitingForRestore;
4453 return self.stop_for_resource();
4454 }
4455 let mut why: Option<String> = None;
4456 match &out {
4457 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4458 why = Some("the fixer produced nothing usable".to_owned());
4459 }
4460 AgentOutcome::Ok(_) => {}
4461 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4462 AgentOutcome::Quota(o) => {
4463 self.state.quota.push(QuotaLoss {
4464 seat: seat.key.clone(),
4465 node: "graft".to_owned(),
4466 at: Timestamp::now(),
4467 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4468 });
4469 why = Some("rate limited (quota)".to_owned());
4470 }
4471 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4472 why = Some(format!(
4473 "the fixer failed: {}",
4474 e.lines()
4475 .next()
4476 .unwrap_or("")
4477 .chars()
4478 .take(160)
4479 .collect::<String>()
4480 ));
4481 }
4482 }
4483 let agent_id = seat.agent.clone();
4484 self.state.seats.insert(seat.key.clone(), seat);
4485
4486 let mut post = None;
4487 let mut lines = None;
4488 let _coordination = self.coordinate(&wt).await?;
4489 if why.is_none() {
4490 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4491 if let Ok(r) = &rescued {
4492 self.state.note_withheld("graft", &r.withheld);
4493 }
4494 let head = git::rev_parse(&wt, "HEAD").await?;
4495 let dirty = match &rescued {
4499 Ok(r) => git::status_porcelain(&wt)
4500 .await
4501 .map(|st| {
4502 st.lines().any(|l| {
4503 !r.withheld
4504 .iter()
4505 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4506 })
4507 })
4508 .unwrap_or(true),
4509 Err(_) => true,
4510 };
4511 if dirty {
4512 why = Some("the fixer left changes that could not be committed".to_owned());
4513 } else if head == pre {
4514 why = Some("the fixer changed nothing".to_owned());
4515 } else {
4516 lines = git::changed_lines(&wt, &pre, &head).await;
4517 post = Some(head);
4518 }
4519 }
4520 let (adoption, adoption_error) = match &out {
4521 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4522 _ => (None, None),
4523 };
4524 let requested = graft::requested(&plan);
4525 let outcome = match why {
4526 None => {
4527 self.state.event(
4528 "graft",
4529 format!(
4530 "applied: {}",
4531 lines.map_or_else(
4532 || "changed lines unavailable".to_owned(),
4533 |n| format!("{n} changed lines")
4534 )
4535 ),
4536 );
4537 GraftOutcome::Applied
4538 }
4539 Some(why) => {
4540 crate::recovery::restore_repair(&wt, &pre)
4543 .await
4544 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4545 self.state.event(
4546 "graft",
4547 format!("rolled back: {why}; the winner is untouched"),
4548 );
4549 GraftOutcome::RolledBack(why)
4550 }
4551 };
4552 self.state.graft = Some(GraftRecord {
4553 pre,
4554 post,
4555 lines,
4556 limit: None,
4557 agent: Some(agent_id),
4558 outcome,
4559 requested,
4560 adoption,
4561 adoption_error,
4562 });
4563 self.state.save()?;
4564 Ok(())
4565 }
4566
4567 async fn fold_losers(&mut self) -> Result<()> {
4570 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4571 return Ok(());
4572 };
4573 if !self.state.candidate_preservation.is_empty() {
4574 bail!("candidate preservation incomplete; refusing cleanup");
4575 }
4576 let repo = self.state.repo.clone();
4577 let _coordination = self.coordinate(&repo).await?;
4578 let mut folded = Vec::new();
4579 for i in 0..self.state.candidates.len() {
4580 let c = &self.state.candidates[i];
4581 if c.label == winner || c.folded {
4582 continue;
4583 }
4584 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4585 remove_preserved_worktree(&repo, &wt).await?;
4586 if git::branch_exists(&repo, &branch).await?
4587 && !git::branch_delete(&repo, &branch).await?
4588 {
4589 bail!("candidate branch removal refused");
4590 }
4591 self.state.candidates[i].folded = true;
4592 self.state.save()?;
4593 folded.push(label.to_string());
4594 }
4595 let root = self.state.worktree_root();
4597 for j in 1..=self.roles.judges.len() {
4598 let wt = root.join(format!("judge-{j}"));
4599 if wt.exists() {
4600 remove_preserved_worktree(&repo, &wt).await?;
4601 }
4602 }
4603 if self.state.config.graph.advise {
4606 for k in 1..=self.state.config.graph.advisors {
4607 let wt = root.join(format!("advisor-{k}"));
4608 if wt.exists() {
4609 remove_preserved_worktree(&repo, &wt).await?;
4610 }
4611 }
4612 }
4613 if !folded.is_empty() {
4614 self.state
4615 .event("fold", format!("folded candidates {}", folded.join(", ")));
4616 self.state.save()?;
4617 }
4618 Ok(())
4619 }
4620
4621 async fn sync_to_base(&mut self) -> Result<()> {
4661 if self.state.status == RunStatus::AlreadyInBase {
4662 return Ok(());
4663 }
4664 let conflicted = self
4665 .state
4666 .base_sync
4667 .as_ref()
4668 .is_some_and(|s| s.conflict.is_some());
4669 let Some(winner) = self.state.winner().cloned() else {
4670 return Ok(());
4671 };
4672
4673 let repo = self.state.repo.clone();
4674 let remote = self.state.config.merge.remote.clone();
4675 let base_branch = self.state.base_branch.clone();
4676 let tracking = format!("{remote}/{base_branch}");
4677
4678 git::fetch(&repo, &remote, &base_branch).await.ok();
4679 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4684 return Ok(());
4685 };
4686
4687 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4688 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4689 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4690
4691 if (behind > 0 || conflicted || head == tip)
4700 && self
4701 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4702 .await?
4703 {
4704 return Ok(());
4705 }
4706 if conflicted {
4707 return Ok(());
4708 }
4709
4710 if behind == 0 {
4711 if let Some(from) = self
4720 .state
4721 .rebase_fixes
4722 .iter()
4723 .rev()
4724 .find_map(|r| r.from.clone())
4725 && from != head
4726 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4727 .await
4728 .is_ok_and(|o| o.ok())
4729 {
4730 git::sync_to_head(&winner.worktree).await?;
4731 }
4732 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4738 if let Some(why) = &conflict {
4739 self.state.status = RunStatus::Blocked;
4740 self.state.event("land", why.clone());
4741 }
4742 self.state.base_sync = Some(BaseSync {
4743 tip,
4744 behind: 0,
4745 attempts,
4746 conflict,
4747 already_in: None,
4748 });
4749 self.state.save()?;
4750 return Ok(());
4751 }
4752
4753 if attempts >= BASE_SYNC_ROUNDS {
4754 let why = format!(
4755 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4756 rebase(s); rebasing again would only race it",
4757 winner.branch
4758 );
4759 self.state.status = RunStatus::Blocked;
4760 self.state.base_sync = Some(BaseSync {
4761 tip,
4762 behind,
4763 attempts,
4764 conflict: Some(why.clone()),
4765 already_in: None,
4766 });
4767 self.state.event("land", why);
4768 self.state.save()?;
4769 return Ok(());
4770 }
4771
4772 self.state.event(
4773 "land",
4774 format!(
4775 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4776 winner.branch
4777 ),
4778 );
4779 self.state.save()?;
4780
4781 let branch_tracking = format!("{remote}/{}", winner.branch);
4787 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4788 let remote_tip = if self.state.task_pr_branch().is_none()
4789 && matches!(&fetched_branch, Ok(o) if o.ok())
4790 {
4791 git::rev_parse(&repo, &branch_tracking).await.ok()
4792 } else {
4793 None
4794 };
4795 if let Some(theirs) = &remote_tip
4799 && !git::is_ancestor(&repo, theirs, &head).await
4800 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4801 .await
4802 .is_ok_and(|missing| missing.is_empty())
4803 {
4804 let why = format!(
4805 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4806 them",
4807 short(theirs),
4808 winner.branch
4809 );
4810 self.state.status = RunStatus::Blocked;
4811 self.state.base_sync = Some(BaseSync {
4812 tip,
4813 behind,
4814 attempts,
4815 conflict: Some(why.clone()),
4816 already_in: None,
4817 });
4818 self.state.event("land", why);
4819 self.state.save()?;
4820 return Ok(());
4821 }
4822
4823 let scratch = self.state.dir().join("base-sync");
4824 let rebased = match crate::rebase::rebase_with_fixer(
4825 &mut self.state,
4826 &scratch,
4827 &winner.branch,
4828 &tracking,
4829 )
4830 .await
4831 {
4832 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4833 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4834 Err(e) => Err(e),
4835 };
4836 let attempts = attempts + 1;
4837 match rebased {
4838 Ok(None) => {
4839 git::sync_to_head(&winner.worktree).await?;
4843 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4844 let mut conflict = None;
4845 if let Some(pinned) = &remote_tip {
4846 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4847 match pushed {
4848 Ok(o) if o.ok() => self.state.event(
4849 "land",
4850 format!("pushed rebased {} to {remote}", winner.branch),
4851 ),
4852 Ok(o) => {
4853 conflict = Some(format!(
4854 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4855 winner.branch,
4856 short(pinned),
4857 o.stderr.chars().take(600).collect::<String>()
4858 ));
4859 }
4860 Err(e) => {
4861 conflict = Some(format!(
4862 "rebased {} locally but could not push it: {e:#}",
4863 winner.branch
4864 ));
4865 }
4866 }
4867 }
4868 if let Some(why) = &conflict {
4869 self.state.status = RunStatus::Blocked;
4870 self.state.event("land", why.clone());
4871 }
4872 self.state.base_sync = Some(BaseSync {
4873 tip: tip.clone(),
4874 behind: 0,
4875 attempts,
4876 conflict,
4877 already_in: None,
4878 });
4879 self.state
4880 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4881 }
4882 Ok(Some(conflict)) => {
4883 let why = format!(
4884 "{} conflicts with {tracking} and did not rebase: {}",
4885 winner.branch,
4886 conflict.chars().take(600).collect::<String>()
4887 );
4888 self.state.status = RunStatus::Blocked;
4889 self.state.base_sync = Some(BaseSync {
4890 tip,
4891 behind,
4892 attempts,
4893 conflict: Some(why.clone()),
4894 already_in: None,
4895 });
4896 self.state.event("land", why);
4897 }
4898 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4902 self.state.save()?;
4903 return Ok(());
4904 }
4905 Err(e) => {
4906 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4907 self.state.status = RunStatus::Blocked;
4908 self.state.base_sync = Some(BaseSync {
4909 tip,
4910 behind,
4911 attempts,
4912 conflict: Some(why.clone()),
4913 already_in: None,
4914 });
4915 self.state.event("land", why);
4916 }
4917 }
4918 self.state.save()?;
4919 Ok(())
4920 }
4921
4922 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4928 if self.state.task_pr_branch().is_some() {
4931 return None;
4932 }
4933 let repo = self.state.repo.clone();
4934 let remote = self.state.config.merge.remote.clone();
4935 let fetched = git::fetch(&repo, &remote, branch).await;
4936 if !matches!(&fetched, Ok(o) if o.ok()) {
4937 return None;
4938 }
4939 let branch_tracking = format!("{remote}/{branch}");
4940 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4941 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4942 return None;
4943 }
4944 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4945 .await
4946 .is_ok_and(|missing| missing.is_empty())
4947 {
4948 return Some(format!(
4949 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4950 them",
4951 short(&theirs)
4952 ));
4953 }
4954 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4955 Ok(o) if o.ok() => {
4956 self.state
4957 .event("land", format!("pushed rebased {branch} to {remote}"));
4958 None
4959 }
4960 Ok(o) => Some(format!(
4961 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4962 someone may have pushed): {}",
4963 short(&theirs),
4964 o.stderr.chars().take(600).collect::<String>()
4965 )),
4966 Err(e) => Some(format!(
4967 "{branch} is rebased locally but could not be pushed: {e:#}"
4968 )),
4969 }
4970 }
4971
4972 async fn settle_already_in(
4989 &mut self,
4990 branch: &str,
4991 tip: &str,
4992 head: &str,
4993 attempts: usize,
4994 behind: usize,
4995 ) -> Result<bool> {
4996 let repo = self.state.repo.clone();
4997 let remote = self.state.config.merge.remote.clone();
4998 let start = self.state.base_commit.clone();
4999 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5000 Ok(Some(e)) => e,
5001 Ok(None) => return Ok(false),
5002 Err(e) => {
5003 tracing::warn!("already-in-base check for {branch}: {e:#}");
5004 return Ok(false);
5005 }
5006 };
5007 let mut verified = vec![head.to_owned()];
5008 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5009 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5010 if matches!(&fetched, Ok(o) if o.ok())
5011 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5012 && theirs != head
5013 {
5014 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5015 Ok(Some(_)) => verified.push(theirs),
5016 _ => return Ok(false),
5017 }
5018 }
5019 let base_branch = self.state.base_branch.clone();
5020 let message = format!(
5021 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5022 land",
5023 evidence.names(),
5024 evidence.proof.as_str()
5025 );
5026 let closed =
5027 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5028 .await;
5029 match closed {
5030 Ok(Ok(url)) => self
5031 .state
5032 .event("land", format!("closed {url}: superseded on {base_branch}")),
5033 Ok(Err(why)) => self
5034 .state
5035 .event("land", format!("did not close a pull request: {why}")),
5036 Err(e) => {
5037 let why = format!(
5038 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5039 not be closed ({e:#}); resume to retry"
5040 );
5041 self.state.status = RunStatus::Blocked;
5042 self.state.base_sync = Some(BaseSync {
5043 tip: tip.to_owned(),
5044 behind,
5045 attempts,
5046 conflict: Some(why.clone()),
5047 already_in: None,
5048 });
5049 self.state.event("land", why);
5050 self.state.save()?;
5051 return Ok(true);
5052 }
5053 }
5054 self.state.status = RunStatus::AlreadyInBase;
5055 self.state.base_sync = Some(BaseSync {
5056 tip: tip.to_owned(),
5057 behind,
5058 attempts,
5059 conflict: None,
5060 already_in: Some(evidence),
5061 });
5062 self.state.event("land", message);
5063 self.state.save()?;
5064 self.settle_questions();
5065 Ok(true)
5066 }
5067
5068 fn landing_base(&self) -> String {
5078 self.state
5079 .base_sync
5080 .as_ref()
5081 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5082 }
5083
5084 pub async fn fix_selected(
5117 &mut self,
5118 ids: &[String],
5119 reason: &str,
5120 allow_stale: bool,
5121 ) -> Result<()> {
5122 let reason = reason.trim();
5123 if reason.is_empty() {
5124 bail!("a fix request needs a reason — that is the operator's own record of why");
5125 }
5126 if ids.is_empty() {
5127 bail!("no finding id given");
5128 }
5129 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5130 bail!(
5131 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5132 has already concluded — can be given a targeted fix. A run still \
5133 in progress should simply be resumed; a `merged` run's branch has \
5134 already landed, so its answer is a fresh `magi review <branch>`, \
5135 not reopening this run's own record",
5136 self.state.id,
5137 self.state.status.as_str()
5138 );
5139 }
5140 let Some(winner) = self.state.winner().cloned() else {
5141 bail!("run {} has no winning candidate to fix", self.state.id);
5142 };
5143 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5144 bail!(
5145 "branch `{}` no longer exists; this run cannot be extended",
5146 winner.branch
5147 );
5148 }
5149 let home = crate::run::home();
5150 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5151 bail!(
5152 "run {} is currently being worked on by another magi process",
5153 self.state.id
5154 );
5155 }
5156 let _claim = FixClaim::acquire(&self.state.dir())?;
5162
5163 let mut seen = BTreeSet::new();
5167 let mut findings = Vec::new();
5168 let mut missing = Vec::new();
5169 for id in ids {
5170 if !seen.insert(id.clone()) {
5171 continue;
5172 }
5173 match self.state.finding(id) {
5174 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5175 id: f.id.clone(),
5176 severity: f.severity,
5177 reviewer_vote: rec.vote,
5178 round: round.round,
5179 round_head: round.head.clone(),
5180 reviewer: rec.reviewer,
5181 agent: rec.agent.clone(),
5182 file: f.file.clone(),
5183 line: f.line,
5184 title: f.title.clone(),
5185 detail: f.detail.clone(),
5186 outcome: OperatorFixOutcome::Pending,
5187 }),
5188 None => missing.push(id.clone()),
5189 }
5190 }
5191 if !missing.is_empty() {
5192 bail!(
5193 "unknown finding id(s): {}; nothing was changed",
5194 missing.join(", ")
5195 );
5196 }
5197
5198 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5199 let stale_details: Vec<(String, String)> = findings
5200 .iter()
5201 .filter(|f| f.round_head != head_at_request)
5202 .map(|f| (f.id.clone(), f.round_head.clone()))
5203 .collect();
5204 let stale = !stale_details.is_empty();
5205 if stale && !allow_stale {
5206 bail!(
5207 "the branch has moved since some finding(s) were raised — {} — now \
5208 at {}; pass --allow-stale to fix anyway, or re-run review first",
5209 stale_details
5210 .iter()
5211 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5212 .collect::<Vec<_>>()
5213 .join(", "),
5214 short(&head_at_request)
5215 );
5216 }
5217
5218 let request = OperatorFixRequest {
5219 requested_at: Timestamp::now(),
5220 reason: reason.to_owned(),
5221 findings,
5222 head_at_request: head_at_request.clone(),
5223 allow_stale,
5224 stale,
5225 fix: None,
5226 result_head: None,
5227 follow_up_review_run: None,
5228 };
5229 self.state.event(
5230 "fix",
5231 format!(
5232 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5233 request.findings.len(),
5234 request
5235 .findings
5236 .iter()
5237 .map(|f| f.id.as_str())
5238 .collect::<Vec<_>>()
5239 .join(", "),
5240 ),
5241 );
5242 self.state.operator_fixes.push(request);
5249 self.state.save()?;
5250 let request_index = self.state.operator_fixes.len() - 1;
5251
5252 let repo = self.state.repo.clone();
5261 let coordination = self.coordinate(&repo).await?;
5262 if winner.worktree.exists() {
5263 let dirty = git::git(
5266 &winner.worktree,
5267 &["status", "--porcelain", "--untracked-files=all"],
5268 )
5269 .await?;
5270 let only_withheld = dirty.lines().all(|l| {
5271 l.strip_prefix("?? ")
5272 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5273 });
5274 if !only_withheld {
5275 bail!(
5276 "`{}` has uncommitted changes; refusing to touch it — commit or \
5277 discard them first",
5278 winner.worktree.display()
5279 );
5280 }
5281 git::worktree_remove(&self.state.repo, &winner.worktree)
5282 .await
5283 .ok();
5284 }
5285 let fix_worktree = self.state.worktree_root().join("operator-fix");
5286 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5287 git::git(
5288 &self.state.repo,
5289 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5290 )
5291 .await
5292 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5293 if !git::is_clean(&fix_worktree).await? {
5294 git::worktree_remove(&self.state.repo, &fix_worktree)
5295 .await
5296 .ok();
5297 bail!(
5298 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5299 winner.branch
5300 );
5301 }
5302 let repo_for_setup = self.state.repo.clone();
5303 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5304 Ok(()) => {
5305 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5306 }
5307 Err(e) => Err(e),
5308 };
5309 if let Err(e) = setup {
5310 git::worktree_remove(&self.state.repo, &fix_worktree)
5311 .await
5312 .ok();
5313 return Err(e);
5314 }
5315
5316 let run_id = self.state.id.clone();
5317 let prompts = self.state.config.prompts.clone();
5318 let language = self.state.config.graph.language.clone();
5319 let sessions = self.state.config.graph.sessions;
5320 let artifacts = agent::artifacts_dir(&self.state.dir());
5321 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5322 .findings
5323 .iter()
5324 .map(|f| Finding {
5325 id: f.id.clone(),
5326 severity: f.severity,
5327 file: f.file.clone(),
5328 line: f.line,
5329 title: f.title.clone(),
5330 detail: f.detail.clone(),
5331 })
5332 .collect();
5333 let fix_prompt = prompt::operator_fix_in(
5334 &self.state.instruction,
5335 &finding_list,
5336 reason,
5337 &stale_details,
5338 &head_at_request,
5339 &language,
5340 &self.state.config.graph.github_language,
5341 );
5342 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5343 drop(coordination);
5344 let (job, seat, out) = self
5345 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5346 prompt: fix_prompt.clone(),
5347 spec,
5348 seat,
5349 cwd: fix_worktree.clone(),
5350 timeout,
5351 allow_write: true,
5352 sessions,
5353 artifacts: artifacts.clone(),
5354 stem: "operator-fix".to_owned(),
5355 handover: None,
5356 })
5357 .await;
5358 let agent_id = seat.agent.clone();
5359
5360 let mut fix = FixRecord {
5361 agent: agent_id,
5362 addressed: Vec::new(),
5363 rejected: Vec::new(),
5364 notes: String::new(),
5365 committed: false,
5366 failed: None,
5367 duration_ms: 0,
5368 continuation: None,
5369 };
5370 let mut final_seat = seat.clone();
5371 match out {
5372 AgentOutcome::Ok(o) => {
5373 fix.duration_ms = o.duration_ms;
5374 let parsed = verdict::extract_json::<FixReport>(&o.text);
5375 let incomplete_reason = match &parsed {
5376 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5377 "the reply parsed, but it reported a command whose own CLI \
5378 never confirmed an exit status"
5379 .to_owned(),
5380 ),
5381 Ok(_) => None,
5382 Err(e) => Some(e.to_string()),
5383 };
5384 match incomplete_reason {
5385 None => {
5386 let report = parsed.expect("checked Ok above");
5387 fix.addressed = report.addressed;
5388 fix.rejected = report.rejected;
5389 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5390 }
5391 Some(reason) => {
5392 let (resumed_seat, resolved, failure, cont) = self
5393 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5394 .await;
5395 fix.duration_ms += cont.cumulative_wait_ms;
5396 fix.continuation = Some(cont);
5397 final_seat = resumed_seat;
5398 match resolved {
5399 Some(report) => {
5400 fix.addressed = report.addressed;
5401 fix.rejected = report.rejected;
5402 fix.notes =
5403 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5404 }
5405 None => fix.failed = failure,
5406 }
5407 }
5408 }
5409 }
5410 AgentOutcome::Dropped(o) => {
5411 fix.duration_ms = o.duration_ms;
5412 let why = o
5413 .dropped
5414 .as_ref()
5415 .map(|d| d.why.as_str())
5416 .unwrap_or("the CLI ended the stream without delivering its answer");
5417 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5418 }
5419 AgentOutcome::Quota(o) => {
5420 self.state.quota.push(QuotaLoss {
5421 seat: final_seat.key.clone(),
5422 node: "fix".to_owned(),
5423 at: Timestamp::now(),
5424 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5425 });
5426 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5427 }
5428 AgentOutcome::Failed(e) => fix.failed = Some(e),
5429 AgentOutcome::ResourceBlocked(e) => {
5430 fix.failed = Some(format!("the fixer was not started: {e}"));
5431 }
5432 }
5433 if fix.continuation.is_none() {
5434 fix.continuation = Some(ContinuationRecord::not_needed());
5435 }
5436 self.state.seats.insert(final_seat.key.clone(), final_seat);
5437
5438 let rescue_message = format!(
5439 "magi: operator-selected fix ({}) (uncommitted work)",
5440 self.state.operator_fixes[request_index]
5441 .findings
5442 .iter()
5443 .map(|f| f.id.as_str())
5444 .collect::<Vec<_>>()
5445 .join(", ")
5446 );
5447 let repo = self.state.repo.clone();
5448 let coordination = self.coordinate(&repo).await?;
5449 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5450 self.state.note_withheld("fix", &r.withheld);
5451 }
5452 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5453 fix.committed = after != head_at_request;
5454 git::worktree_remove(&self.state.repo, &fix_worktree)
5455 .await
5456 .ok();
5457 drop(coordination);
5458
5459 self.state.event(
5460 "fix",
5461 match &fix.failed {
5462 Some(reason) => format!(
5463 "operator fix: adoption report was lost ({reason}); {}",
5464 if fix.committed {
5465 "committed"
5466 } else {
5467 "NO new commit"
5468 }
5469 ),
5470 None => format!(
5471 "operator fix: {} addressed, {} rejected, {}",
5472 fix.addressed.len(),
5473 fix.rejected.len(),
5474 if fix.committed {
5475 "committed"
5476 } else {
5477 "NO new commit"
5478 }
5479 ),
5480 },
5481 );
5482
5483 for f in &mut self.state.operator_fixes[request_index].findings {
5490 f.outcome = if fix.failed.is_some() {
5491 OperatorFixOutcome::Unreported
5492 } else if fix.addressed.contains(&f.id) {
5493 OperatorFixOutcome::Addressed
5494 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5495 OperatorFixOutcome::Rejected { why: r.why.clone() }
5496 } else {
5497 OperatorFixOutcome::Unreported
5498 };
5499 }
5500
5501 let committed = fix.committed;
5502 if committed {
5503 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5504 }
5505 self.state.operator_fixes[request_index].fix = Some(fix);
5506 self.state.save()?;
5509
5510 if committed {
5511 self.state.event(
5512 "fix",
5513 format!(
5514 "operator fix committed {}; opening a follow-up review-only run",
5515 short(&after)
5516 ),
5517 );
5518 let origin =
5521 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5522 match Self::review(
5523 &self.state.repo,
5524 &winner.branch,
5525 self.state.config.clone(),
5526 origin,
5527 )
5528 .await
5529 {
5530 Ok(mut follow_up) => {
5531 follow_up.state.event(
5532 "start",
5533 format!(
5534 "requested by an operator fix on run {} for finding(s) {}",
5535 self.state.id,
5536 self.state.operator_fixes[request_index]
5537 .findings
5538 .iter()
5539 .map(|f| f.id.as_str())
5540 .collect::<Vec<_>>()
5541 .join(", "),
5542 ),
5543 );
5544 follow_up.state.save()?;
5545 let follow_up_id = follow_up.state.id.clone();
5546 let adopted = match crate::direct::adopt(
5549 &follow_up.state,
5550 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5551 ) {
5552 Ok(a) => a,
5553 Err(e) => {
5554 self.state.event(
5557 "fix",
5558 format!(
5559 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5560 ),
5561 );
5562 self.state.save()?;
5563 return Ok(());
5564 }
5565 };
5566 let executed = follow_up.execute().await;
5567 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5568 if let Some(a) = adopted {
5569 a.finish(&follow_up.state, executed);
5570 }
5571 if let Some(e) = failure {
5572 self.state.event(
5573 "fix",
5574 format!(
5575 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5576 ),
5577 );
5578 }
5579 self.state.operator_fixes[request_index].follow_up_review_run =
5580 Some(follow_up_id);
5581 }
5582 Err(e) => {
5583 self.state.event(
5584 "fix",
5585 format!("committed the fix but could not open a follow-up review: {e:#}"),
5586 );
5587 }
5588 }
5589 self.state.save()?;
5590 }
5591
5592 Ok(())
5593 }
5594
5595 async fn ask_fixer(
5615 &mut self,
5616 winner: &Candidate,
5617 node: &'static str,
5618 round: Option<usize>,
5619 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5620 ) -> (SeatJob, SeatState, AgentOutcome) {
5621 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5622 self.ask_fixer_among(attempts, node, round, None, build)
5623 .await
5624 }
5625
5626 async fn ask_fixer_among(
5631 &mut self,
5632 attempts: Vec<(AgentSpec, String)>,
5633 node: &'static str,
5634 round: Option<usize>,
5635 restore: Option<(&Path, &str)>,
5636 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5637 ) -> (SeatJob, SeatState, AgentOutcome) {
5638 let run_id = self.state.id.clone();
5639 let prompts = self.state.config.prompts.clone();
5640 let cache = self.state.config.cache_dir();
5641 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5642 let mut last = None;
5643 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5644 let seat = self.seat(&key, &spec.id);
5645 let mut job = build(spec.clone(), seat);
5646 if i > 0 {
5647 job.stem = format!("{}-{}", job.stem, spec.id);
5648 }
5649 let ctx = WaveCtx {
5650 carry_seats: false,
5651 pauses: Some((&self.pause, &self.interrupt)),
5652 run: &run_id,
5653 node,
5654 prompts: &prompts,
5655 cache: cache.as_deref(),
5656 round,
5657 };
5658 let (seat, out) =
5659 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5660 let advances = match &out {
5663 AgentOutcome::Ok(o) => agent::output_advances(o),
5664 AgentOutcome::ResourceBlocked(_) => false,
5665 _ => true,
5666 };
5667 if let Some(next) = ids.get(i + 1)
5668 && advances
5669 {
5670 self.state.seats.insert(seat.key.clone(), seat.clone());
5671 let class =
5672 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5673 let reason = match &out {
5674 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5675 other => fail_reason(other),
5676 };
5677 if let Some((wt, rev)) = restore
5681 && let Err(e) = self.graft_restore(wt, rev).await
5682 {
5683 let why =
5684 format!("could not restore the worktree before the next fixer: {e:#}");
5685 tracing::warn!("{why}");
5686 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5687 AgentOutcome::ResourceBlocked(why)
5688 } else {
5689 AgentOutcome::Failed(why)
5690 };
5691 last = Some((job, seat, out));
5692 break;
5693 }
5694 record_handover(
5695 &mut self.state,
5696 node,
5697 &seat.key,
5698 &spec.id,
5699 next,
5700 &class,
5701 &reason,
5702 );
5703 continue;
5704 }
5705 last = Some((job, seat, out));
5706 break;
5707 }
5708 last.expect("the fixer chain always has an entry")
5709 }
5710
5711 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5722 use crate::recovery::{Mode, Repair};
5723 if !self.state.reviews.is_empty() {
5724 return Ok(());
5725 }
5726 let Some(mut rec) = self.state.recovery.clone() else {
5727 return Ok(());
5728 };
5729 if rec.mode != Mode::VerifyRepair {
5730 return Ok(());
5731 }
5732 let wt = winner.worktree.clone();
5733 let mut head = git::rev_parse(&wt, "HEAD").await?;
5734 match rec.repair.clone() {
5735 Repair::Done { .. } => return Ok(()),
5736 Repair::WaitingForRestore { before } => {
5737 self.graft_restore(&wt, &before).await?;
5738 head = before;
5739 rec.repair = Repair::NotStarted;
5740 self.state.recovery = Some(rec.clone());
5741 self.state.save()?;
5742 }
5743 Repair::Started { before } => {
5744 if let Err(e) = self.graft_restore(&wt, &before).await {
5748 if e.is::<crate::repo_lock::ResourceBlocked>() {
5749 return Err(e);
5750 }
5751 return self
5752 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5753 .await;
5754 }
5755 rec.repair = Repair::Done {
5756 before: before.clone(),
5757 after: before,
5758 committed: false,
5759 failed: Some("interrupted; rolled back".to_owned()),
5760 };
5761 self.state
5762 .event("recovery", "the repair was interrupted and rolled back");
5763 self.state.recovery = Some(rec);
5764 self.state.save()?;
5765 return Ok(());
5766 }
5767 Repair::NotStarted => {}
5768 }
5769 if crate::recovery::recheck(&mut rec, &head) {
5770 self.state.event("recovery", rec.summary());
5771 self.state.recovery = Some(rec);
5772 self.state.save()?;
5773 return Ok(());
5774 }
5775 rec.repair = Repair::Started {
5776 before: head.clone(),
5777 };
5778 self.state.recovery = Some(rec.clone());
5779 self.state.save()?;
5780
5781 let language = self.state.config.graph.language.clone();
5782 let sessions = self.state.config.graph.sessions;
5783 let artifacts = agent::artifacts_dir(&self.state.dir());
5784 let fix_prompt = prompt::recovery_fix(
5785 &self.state.instruction,
5786 &rec,
5787 self.state.config.graph.review_rounds,
5788 &language,
5789 );
5790 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5791 self.state.event(
5792 "recovery",
5793 format!(
5794 "repair: the fixer starts from the failed verification on {}",
5795 short(&head)
5796 ),
5797 );
5798 let (_job, seat, out) = self
5799 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5800 prompt: fix_prompt.clone(),
5801 spec,
5802 seat,
5803 cwd: wt.clone(),
5804 timeout,
5805 allow_write: true,
5806 sessions,
5807 artifacts: artifacts.clone(),
5808 stem: "fix-recovery".to_owned(),
5809 handover: None,
5810 })
5811 .await;
5812 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5813 rec.repair = Repair::WaitingForRestore { before: head };
5814 self.state.recovery = Some(rec);
5815 return self.stop_for_resource();
5816 }
5817 let failed = match &out {
5818 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5819 AgentOutcome::Quota(o) => {
5820 self.state.quota.push(QuotaLoss {
5821 seat: seat.key.clone(),
5822 node: "fix".to_owned(),
5823 at: Timestamp::now(),
5824 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5825 });
5826 Some("rate limited (quota); the fixer could not run".to_owned())
5827 }
5828 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5829 AgentOutcome::Failed(e) => Some(e.clone()),
5830 };
5831 self.state.seats.insert(seat.key.clone(), seat);
5832 let coordination = self.coordinate(&wt).await?;
5833 let failed = match failed {
5834 Some(why) => {
5836 if let Err(e) = git::restore_to(&wt, &head).await {
5837 return self
5838 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5839 .await;
5840 }
5841 Some(why)
5842 }
5843 None => {
5844 if let Ok(r) = git::rescue_commit(
5845 &wt,
5846 "magi: recovery repair of the failed verification (uncommitted work)",
5847 )
5848 .await
5849 {
5850 self.state.note_withheld("fix", &r.withheld);
5851 }
5852 None
5853 }
5854 };
5855 let after = git::rev_parse(&wt, "HEAD").await?;
5856 drop(coordination);
5857 self.state.event(
5858 "recovery",
5859 match &failed {
5860 Some(why) => format!("repair: nothing kept ({why})"),
5861 None if after != head => {
5862 format!("repair: committed {}", short(&after))
5863 }
5864 None => "repair: the fixer made no commit".to_owned(),
5865 },
5866 );
5867 rec.repair = Repair::Done {
5868 before: head.clone(),
5869 committed: after != head,
5870 after,
5871 failed,
5872 };
5873 self.state.recovery = Some(rec);
5874 self.state.save()?;
5875 Ok(())
5876 }
5877
5878 async fn stop_recovery(
5881 &mut self,
5882 mut rec: crate::recovery::RecoveryRecord,
5883 head: &str,
5884 why: String,
5885 ) -> Result<()> {
5886 rec.repair = crate::recovery::Repair::Started {
5888 before: head.to_owned(),
5889 };
5890 self.state.recovery = Some(rec);
5891 self.state
5892 .event("recovery", format!("repair could not be undone: {why}"));
5893 self.state.status = RunStatus::Blocked;
5894 self.state.save()?;
5895 bail!("recovery repair could not be rolled back: {why}")
5896 }
5897
5898 async fn review_loop(&mut self) -> Result<()> {
5899 loop {
5900 if self
5901 .state
5902 .base_sync
5903 .as_ref()
5904 .is_some_and(|s| s.conflict.is_some())
5905 {
5906 return Ok(());
5907 }
5908 if self.catchup_repairable() {
5909 let record = self.state.reviews.last().expect("repairable round").clone();
5910 let winner = self
5911 .state
5912 .winner()
5913 .cloned()
5914 .context("catch-up repair has no winner")?;
5915 if !self.fix_review_round(&winner, record).await? {
5916 return Ok(());
5917 }
5918 }
5919 self.review_loop_pass().await?;
5920 if !self.catchup_repairable() {
5921 return Ok(());
5922 }
5923 }
5924 }
5925
5926 fn catchup_repairable(&self) -> bool {
5929 self.state.reviews.len() < self.state.config.graph.review_rounds
5930 && self.state.reviews.last().is_some_and(|r| {
5931 !r.clean
5932 && r.blocking == 0
5933 && r.fix.is_none()
5934 && r.e2e_status() == E2eStatus::Failed
5935 })
5936 }
5937
5938 async fn review_loop_pass(&mut self) -> Result<()> {
5939 if self
5944 .state
5945 .base_sync
5946 .as_ref()
5947 .is_some_and(|s| s.conflict.is_some())
5948 {
5949 return Ok(());
5950 }
5951 let run_id = self.state.id.clone();
5956 let prompts = self.state.config.prompts.clone();
5957 let Some(winner) = self.state.winner().cloned() else {
5958 return Ok(());
5959 };
5960 let max_rounds = self.state.config.graph.review_rounds;
5961 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5971 if status == RunStatus::Gating {
5974 self.record_contested_handoff();
5975 }
5976 self.state.status = status;
5977 self.state.save()?;
5978 return Ok(());
5979 }
5980 self.state.status = RunStatus::Reviewing;
5981 if self.state.reviews.last().is_some_and(|r| {
5991 r.e2e_status() == E2eStatus::ResourceBlocked
5992 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5993 }) {
5994 let shell = self.state.config.shell();
5995 return self
5996 .stop_unresolved_review(
5997 "the last round's own verification never resolved",
5998 &shell,
5999 &winner.worktree,
6000 )
6001 .await;
6002 }
6003
6004 self.recovery_repair(&winner).await?;
6005 if self.resource_halt()? {
6006 return Ok(());
6007 }
6008
6009 let repo = self.state.repo.clone();
6010 let root = self.state.worktree_root();
6011 let language = self.state.config.graph.language.clone();
6012 let sessions = self.state.config.graph.sessions;
6013 let artifacts = self.merge_repair_artifacts();
6014 let base = self.landing_base();
6015 let base_short = short(&base);
6016 let reviewers = self.roles.reviewers.clone();
6017 let shell = self.state.config.shell();
6018
6019 for round in (self.state.reviews.len() + 1)..=max_rounds {
6020 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6021 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6022 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6023 let prev_verification = self
6032 .state
6033 .reviews
6034 .last()
6035 .and_then(|r| r.verification_summary(&head));
6036 let prior_section = match (&self.state.recovery, round) {
6039 (Some(rec), 1) if self.state.reviews.is_empty() => {
6040 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6041 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6042 Some(crate::recovery::head_of(
6043 &git::diff(&winner.worktree, reviewed, "HEAD")
6044 .await
6045 .unwrap_or_default(),
6046 crate::recovery::SINCE_DIFF_MAX,
6047 ))
6048 }
6049 _ => None,
6050 };
6051 Some(prompt::recovery_review(rec, since.as_deref()))
6052 }
6053 _ => None,
6054 };
6055
6056 let mut jobs = Vec::new();
6060 let coordination = self.coordinate(&repo).await?;
6061 for (r, spec) in reviewers.iter().cloned().enumerate() {
6062 let wt = root.join(format!("review-{}", r + 1));
6063 if wt.exists() {
6066 git::reset_detached(&wt, &head).await?;
6067 } else {
6068 git::worktree_add_detached(&repo, &wt, &head).await?;
6069 }
6070 self.setup_seat_worktree(&repo, &wt, false).await?;
6071 let seat_key = format!("review-{}", r + 1);
6072 let spec = pick_start_spec(
6075 &self.roles.reviewer_roster,
6076 spec,
6077 self.state.seat_history.get(&seat_key),
6078 );
6079 let seat = self.seat(&seat_key, &spec.id);
6080 jobs.push(SeatJob {
6081 prompt: prompt::review(&prompt::ReviewCtx {
6082 instruction: &self.state.instruction,
6083 branch: &winner.branch,
6084 base_short: &base_short,
6085 stat: &stat,
6086 patch: &patch,
6087 verification: prev_verification.as_ref(),
6088 prior: prior_section.as_deref(),
6089 reviewers: reviewers.len(),
6090 round,
6091 rounds: max_rounds,
6092 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6095 lens: Lens::for_seat(r),
6096 language: &language,
6097 github_language: &self.state.config.graph.github_language,
6098 }),
6099 spec,
6100 seat,
6101 cwd: wt,
6102 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6103 allow_write: false,
6104 sessions,
6105 artifacts: artifacts.clone(),
6106 stem: format!("review-{round}-{}", r + 1),
6107 handover: None,
6108 });
6109 }
6110
6111 drop(coordination);
6112 self.state.event(
6113 "review",
6114 format!(
6115 "round {round}: {} reviewers on {}",
6116 jobs.len(),
6117 short(&head)
6118 ),
6119 );
6120 let mut quota_losses = Vec::new();
6121 let review_retries = self.state.config.graph.retries;
6122 let review_cache = self.state.config.cache_dir();
6123 let ctx = WaveCtx {
6124 carry_seats: true,
6125 pauses: Some((&self.pause, &self.interrupt)),
6126 run: &run_id,
6127 node: "review",
6128 prompts: &prompts,
6129 cache: review_cache.as_deref(),
6130 round: Some(round),
6131 };
6132 let results = ask_json_wave::<Review>(
6133 jobs,
6134 Arc::clone(&self.sem),
6135 review_retries,
6136 &self.roles.reviewer_roster,
6137 &ctx,
6138 &mut quota_losses,
6139 &mut self.state,
6140 &|_: &Review| Ok(()),
6141 )
6142 .await;
6143 if self.state.resource_block.is_some() {
6144 return self.stop_for_resource();
6145 }
6146 let round_quota_missing = quota_losses.len();
6150 self.state.quota.extend(quota_losses);
6151
6152 let mut records = Vec::new();
6153 let mut all_findings = Vec::new();
6154 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6155 let agent_id = seat.agent.clone();
6156 self.state.seats.insert(seat.key.clone(), seat);
6157 let mut record = ReviewRecord {
6158 reviewer: r + 1,
6159 agent: agent_id,
6160 summary: String::new(),
6161 findings: Vec::new(),
6162 vote: None,
6163 failed: None,
6164 duration_ms: 0,
6165 attempts,
6171 };
6172 match res {
6173 Ok((review, out)) => {
6174 record.summary =
6182 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6183 record.vote = Some(review.vote);
6184 record.duration_ms = out.duration_ms;
6185 for (n, mut f) in review.findings.into_iter().enumerate() {
6186 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6189 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6190 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6191 f.file = f
6197 .file
6198 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6199 all_findings.push(f.clone());
6200 record.findings.push(f);
6201 }
6202 self.state.event(
6203 "review",
6204 format!(
6205 "round {round}: reviewer {} voted {} with {} finding(s)",
6206 r + 1,
6207 review.vote.label(),
6208 record.findings.len()
6209 ),
6210 );
6211 }
6212 Err(e) => {
6213 record.failed = Some(e.to_string());
6214 self.state.event(
6215 "review",
6216 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6217 );
6218 }
6219 }
6220 records.push(record);
6221 }
6222
6223 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6230 let vote_split =
6231 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6232 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6233 if vote_split {
6234 self.state.event(
6235 "review",
6236 format!(
6237 "round {round}: votes split ({}) — one round of reconsideration",
6238 initial_votes
6239 .iter()
6240 .map(|v| v.label())
6241 .collect::<Vec<_>>()
6242 .join(", ")
6243 ),
6244 );
6245 let panel: Vec<ReviewSeatReport<'_>> = records
6248 .iter()
6249 .filter_map(|r| {
6250 r.vote.map(|vote| ReviewSeatReport {
6251 reviewer: r.reviewer,
6252 vote,
6253 summary: &r.summary,
6254 findings: &r.findings,
6255 })
6256 })
6257 .collect();
6258
6259 let mut jobs = Vec::new();
6260 let mut seats_at = Vec::new();
6261 for (r, spec) in reviewers.iter().cloned().enumerate() {
6262 if records[r].vote.is_none() {
6266 continue;
6267 }
6268 let wt = root.join(format!("review-{}", r + 1));
6269 let seat_key = format!("review-{}", r + 1);
6270 let spec = self.occupant(&seat_key, spec);
6271 let seat = self.seat(&seat_key, &spec.id);
6272 let build = |with_patch: bool| {
6280 prompt::review_reconsider(&ReviewReconsiderCtx {
6281 instruction: &self.state.instruction,
6282 reviewer: r + 1,
6283 lens: Lens::for_seat(r),
6284 panel: &panel,
6285 patch: with_patch.then_some(ReviewPatch {
6286 branch: &winner.branch,
6287 base_short: &base_short,
6288 stat: &stat,
6289 patch: &patch,
6290 }),
6291 round,
6292 rounds: max_rounds,
6293 language: &language,
6294 })
6295 };
6296 let full = build(true);
6297 let prompt = if has_context(&spec, &seat, sessions) {
6298 build(false)
6299 } else {
6300 full.clone()
6301 };
6302 jobs.push(SeatJob {
6303 prompt,
6304 spec,
6305 seat,
6306 cwd: wt,
6307 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6308 allow_write: false,
6309 sessions,
6310 artifacts: artifacts.clone(),
6311 stem: format!("review-{round}-reconsider-{}", r + 1),
6312 handover: Some(full),
6313 });
6314 seats_at.push(r);
6315 }
6316
6317 let mut recon_quota_losses = Vec::new();
6318 let recon_cache = self.state.config.cache_dir();
6319 let recon_ctx = WaveCtx {
6320 carry_seats: true,
6321 pauses: Some((&self.pause, &self.interrupt)),
6322 run: &run_id,
6323 node: "review",
6324 prompts: &prompts,
6325 cache: recon_cache.as_deref(),
6326 round: Some(round),
6327 };
6328 let recon_results = ask_json_wave::<ReviewRevote>(
6329 jobs,
6330 Arc::clone(&self.sem),
6331 review_retries,
6332 &self.roles.reviewer_roster,
6333 &recon_ctx,
6334 &mut recon_quota_losses,
6335 &mut self.state,
6336 &|_: &ReviewRevote| Ok(()),
6337 )
6338 .await;
6339 if self.state.resource_block.is_some() {
6340 return self.stop_for_resource();
6341 }
6342 self.state.quota.extend(recon_quota_losses);
6343
6344 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6345 let agent_id = seat.agent.clone();
6346 self.state.seats.insert(seat.key.clone(), seat);
6347 let mut rec = ReviewRevoteRecord {
6348 reviewer: r + 1,
6349 agent: agent_id,
6350 vote: None,
6351 reason: String::new(),
6352 failed: None,
6353 };
6354 match res {
6355 Ok((rv, _)) => {
6356 rec.vote = Some(rv.vote);
6357 rec.reason =
6358 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6359 self.state.event(
6360 "review",
6361 format!(
6362 "round {round}: reviewer {} revoted {}",
6363 r + 1,
6364 rv.vote.label()
6365 ),
6366 );
6367 }
6368 Err(e) => {
6369 rec.failed = Some(e.to_string());
6370 self.state.event(
6371 "review",
6372 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6373 );
6374 }
6375 }
6376 reconsideration.push(rec);
6377 }
6378 } else if initial_votes.len() > 1 {
6379 self.state.event(
6380 "review",
6381 format!(
6382 "round {round}: votes agreed ({}) — no reconsideration",
6383 initial_votes[0].label()
6384 ),
6385 );
6386 }
6387
6388 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6389 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6390 let defer_e2e =
6401 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6402 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6403 let reason =
6404 format!("{blocking} blocking finding(s) already required a fix this round");
6405 self.state.event(
6406 "verify",
6407 format!(
6408 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6409 {}); it will run once a round has none left",
6410 short(&head)
6411 ),
6412 );
6413 (Vec::new(), false, true, Some(reason))
6414 } else {
6415 let e2e_commands = self.state.config.verify.e2e.clone();
6416 let cache_dir = self.state.config.cache_dir();
6417 let context = format!("round {round}");
6418 let wait = self.repository_wait();
6419 let (e2e, verify_retried) = with_cache_lease(
6420 &mut self.state,
6421 cache_dir.as_deref(),
6422 "e2e",
6423 "e2e",
6424 &winner.worktree,
6425 &head,
6426 verify_timeout,
6427 &context,
6428 &wait,
6429 |state, budget| {
6430 let shell = shell.clone();
6431 let e2e_commands = e2e_commands.clone();
6432 let worktree = winner.worktree.clone();
6433 let context = context.clone();
6434 async move {
6435 run_e2e_with_retry(
6436 state,
6437 &shell,
6438 &e2e_commands,
6439 &worktree,
6440 budget,
6441 &context,
6442 )
6443 .await
6444 }
6445 },
6446 )
6447 .await?;
6448 (e2e, verify_retried, false, None)
6449 };
6450
6451 let expected = records.len();
6452 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6453 let incomplete = answered < expected;
6454 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6455 let policy = self.state.config.graph.incomplete_review;
6456 let clean = round_is_clean(
6457 blocking,
6458 e2e_ok,
6459 answered,
6460 expected,
6461 round_quota_missing,
6462 policy,
6463 );
6464
6465 let mut round_record = ReviewRound {
6466 round,
6467 head: head.clone(),
6468 verified_head: None,
6469 verified_at: None,
6470 reviews: records,
6471 e2e,
6472 verify_retried,
6473 e2e_deferred,
6474 e2e_defer_reason,
6475 fix: None,
6476 blocking,
6477 answered,
6478 expected,
6479 clean,
6480 progressed: false,
6481 vote_split,
6482 reconsideration,
6483 verdict: None,
6484 };
6485 round_record.verdict = ReviewVote::worst(
6489 round_record
6490 .final_votes()
6491 .into_iter()
6492 .map(|(_, _, vote)| vote),
6493 );
6494 if !matches!(
6505 round_record.e2e_status(),
6506 E2eStatus::Deferred | E2eStatus::NotConfigured
6507 ) {
6508 round_record.verified_head = Some(head.clone());
6509 round_record.verified_at = Some(Timestamp::now());
6510 }
6511
6512 if incomplete {
6513 let missing: Vec<String> = round_record
6514 .reviews
6515 .iter()
6516 .filter(|r| r.failed.is_some())
6517 .map(|r| format!("review-{}", r.reviewer))
6518 .collect();
6519 self.state.event(
6520 "review",
6521 format!(
6522 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6523 missing.join(", ")
6524 ),
6525 );
6526 }
6527
6528 if clean {
6529 self.state.event(
6530 "review",
6531 if incomplete && policy == IncompleteReviewPolicy::Warn {
6532 format!(
6533 "round {round}: clean (warn policy, incomplete panel) — no \
6534 blocking findings from the seats that answered, verification green"
6535 )
6536 } else if incomplete {
6537 format!(
6538 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6539 quorum) — no blocking findings from the seats that answered, \
6540 verification green",
6541 expected - answered
6542 )
6543 } else {
6544 format!("round {round}: clean — no blocking findings, verification green")
6545 },
6546 );
6547 self.state.reviews.push(round_record);
6548 self.state.status = RunStatus::Gating;
6549 self.state.save()?;
6550 return Ok(());
6551 }
6552
6553 if incomplete && blocking == 0 && e2e_ok {
6561 self.state.reviews.push(round_record);
6562 self.state.save()?;
6563 if round == max_rounds {
6564 self.state.status = RunStatus::Blocked;
6565 self.state.event(
6566 "review",
6567 format!(
6568 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6569 refusing to call it clean",
6570 expected - answered
6571 ),
6572 );
6573 return Ok(());
6574 }
6575 continue;
6576 }
6577
6578 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6590 self.state.reviews.push(round_record);
6591 return self
6592 .stop_unresolved_review(
6593 "the round's own verification could not run",
6594 &shell,
6595 &winner.worktree,
6596 )
6597 .await;
6598 }
6599
6600 if round == max_rounds {
6601 self.state.reviews.push(round_record);
6602 return self
6603 .stop_reviewing(
6604 &format!(
6605 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6606 ),
6607 &shell,
6608 &winner.worktree,
6609 )
6610 .await;
6611 }
6612
6613 if !self.fix_review_round(&winner, round_record).await? {
6614 return Ok(());
6615 }
6616 }
6617 Ok(())
6618 }
6619
6620 async fn fix_review_round(
6623 &mut self,
6624 winner: &Candidate,
6625 mut round_record: ReviewRound,
6626 ) -> Result<bool> {
6627 let round = round_record.round;
6628 let shell = self.state.config.shell();
6629 let step = self.run_fix_round(winner, &mut round_record).await?;
6630 if matches!(step, FixStep::Resource) {
6631 self.stop_for_resource()?;
6632 return Ok(false);
6633 }
6634 if self.state.reviews.len() < round {
6635 self.state.reviews.push(round_record);
6636 } else {
6637 self.state.reviews[round - 1] = round_record;
6638 }
6639 self.state.save()?;
6640
6641 if matches!(step, FixStep::Recorded { report_lost: true }) {
6654 self.stop_reviewing(
6655 "the fixer's adoption report never came back, even after resuming its \
6656 own seat; refusing to start another round against the same worktree \
6657 while that is unresolved",
6658 &shell,
6659 &winner.worktree,
6660 )
6661 .await?;
6662 return Ok(false);
6663 }
6664
6665 let streak = self
6666 .state
6667 .reviews
6668 .iter()
6669 .rev()
6670 .take_while(|r| !r.progressed)
6671 .count();
6672 if streak >= STAGNANT_LIMIT {
6673 self.stop_reviewing(
6674 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6675 &shell,
6676 &winner.worktree,
6677 )
6678 .await?;
6679 return Ok(false);
6680 }
6681 Ok(true)
6682 }
6683
6684 async fn run_fix_round(
6686 &mut self,
6687 winner: &Candidate,
6688 round_record: &mut ReviewRound,
6689 ) -> Result<FixStep> {
6690 let round = round_record.round;
6691 let max_rounds = self.state.config.graph.review_rounds;
6692 let run_id = self.state.id.clone();
6693 let prompts = self.state.config.prompts.clone();
6694 let language = self.state.config.graph.language.clone();
6695 let sessions = self.state.config.graph.sessions;
6696 let artifacts = self.merge_repair_artifacts();
6697 let base = self.landing_base();
6698 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6699 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6700 let this_round_verification = round_record.verification_summary(&head);
6701 let all_findings: Vec<_> = round_record
6702 .reviews
6703 .iter()
6704 .flat_map(|r| r.findings.iter().cloned())
6705 .collect();
6706 let blocking_findings: Vec<_> = all_findings
6709 .iter()
6710 .filter(|f| f.severity.blocks())
6711 .cloned()
6712 .collect();
6713 let fix_prompt = prompt::fix_in(
6714 &self.state.instruction,
6715 &blocking_findings,
6716 this_round_verification.as_ref(),
6717 round,
6718 max_rounds,
6719 &language,
6720 &self.state.config.graph.github_language,
6721 );
6722 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6723 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6724 let (job, seat, out) = self
6725 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6726 prompt: fix_prompt.clone(),
6727 spec,
6728 seat,
6729 cwd: winner.worktree.clone(),
6730 timeout,
6731 allow_write: true,
6732 sessions,
6733 artifacts: artifacts.clone(),
6734 stem: format!("fix-{round}"),
6735 handover: None,
6736 })
6737 .await;
6738 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6741 return Ok(FixStep::Resource);
6742 }
6743 let agent_id = seat.agent.clone();
6744
6745 let mut fix = FixRecord {
6746 agent: agent_id,
6747 addressed: Vec::new(),
6748 rejected: Vec::new(),
6749 notes: String::new(),
6750 committed: false,
6751 failed: None,
6752 duration_ms: 0,
6753 continuation: None,
6754 };
6755 let mut continuation = ContinuationRecord::not_needed();
6756 let mut final_seat = seat.clone();
6757 match out {
6758 AgentOutcome::Ok(o) => {
6759 fix.duration_ms = o.duration_ms;
6760 let parsed = verdict::extract_json::<FixReport>(&o.text);
6761 let incomplete_reason = match &parsed {
6768 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6769 "the reply parsed, but it reported a command whose own CLI never \
6770 confirmed an exit status"
6771 .to_owned(),
6772 ),
6773 Ok(_) => None,
6774 Err(e) => Some(e.to_string()),
6775 };
6776 match incomplete_reason {
6777 None => {
6778 let report = parsed.expect("checked Ok above");
6779 fix.addressed = report.addressed;
6780 fix.rejected = report.rejected;
6781 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6782 }
6783 Some(reason) => {
6784 let (resumed_seat, resolved, failure, cont) = self
6785 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6786 .await;
6787 fix.duration_ms += cont.cumulative_wait_ms;
6788 continuation = cont;
6789 final_seat = resumed_seat;
6790 match resolved {
6791 Some(report) => {
6792 fix.addressed = report.addressed;
6793 fix.rejected = report.rejected;
6794 fix.notes =
6795 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6796 }
6797 None => fix.failed = failure,
6798 }
6799 }
6800 }
6801 }
6802 AgentOutcome::Dropped(o) => {
6804 fix.duration_ms = o.duration_ms;
6805 let why = o
6806 .dropped
6807 .as_ref()
6808 .map(|d| d.why.as_str())
6809 .unwrap_or("the CLI ended the stream without delivering its answer");
6810 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6811 }
6812 AgentOutcome::Quota(o) => {
6813 self.state.quota.push(QuotaLoss {
6814 seat: final_seat.key.clone(),
6815 node: "fix".to_owned(),
6816 at: Timestamp::now(),
6817 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6818 });
6819 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6820 }
6821 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6822 }
6823 fix.continuation = Some(continuation);
6824 self.state.seats.insert(final_seat.key.clone(), final_seat);
6825 let coordination = self.coordinate(&winner.worktree).await?;
6826 if let Ok(r) = git::rescue_commit(
6827 &winner.worktree,
6828 &format!("magi: review round {round} fixes (uncommitted work)"),
6829 )
6830 .await
6831 {
6832 self.state.note_withheld("fix", &r.withheld);
6833 }
6834 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6835 fix.committed = after != before;
6836 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6844 drop(coordination);
6845 let progressed = diff_after != patch;
6846 let commit_note = if fix.committed {
6847 "committed"
6848 } else {
6849 "NO new commit"
6850 };
6851 let tree_note = if progressed {
6852 "changed vs base"
6853 } else {
6854 "unchanged vs base"
6855 };
6856 self.state.event(
6857 "fix",
6858 match &fix.failed {
6859 Some(reason) => {
6865 format!(
6866 "round {round}: fixer's adoption report was lost ({reason}); \
6867 {commit_note}, tree {tree_note}"
6868 )
6869 }
6870 None => format!(
6871 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6872 {tree_note}{}",
6873 fix.addressed.len(),
6874 fix.rejected.len(),
6875 if continuation.outcome == ContinuationOutcome::Resumed {
6876 format!(
6877 " (adoption report recovered after {} continuation(s))",
6878 continuation.attempts
6879 )
6880 } else {
6881 String::new()
6882 },
6883 ),
6884 },
6885 );
6886 round_record.fix = Some(fix);
6887 round_record.progressed = progressed;
6888 Ok(FixStep::Recorded {
6889 report_lost: matches!(
6890 continuation.outcome,
6891 ContinuationOutcome::Exhausted
6892 | ContinuationOutcome::QuotaLost
6893 | ContinuationOutcome::NoSession
6894 ),
6895 })
6896 }
6897
6898 fn record_contested_handoff(&mut self) {
6932 if self.state.contested_handoff.is_some() {
6933 return;
6934 }
6935 let Some(contested) = self
6936 .state
6937 .reviews
6938 .last()
6939 .and_then(ReviewRound::contested_handoff)
6940 else {
6941 return;
6942 };
6943 self.state.event(
6944 "review",
6945 format!(
6946 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6947 wait for the owner's approval",
6948 contested.findings.len(),
6949 contested.rejecters.len()
6950 ),
6951 );
6952 self.state.contested_handoff = Some(contested);
6953 }
6954
6955 async fn stop_unresolved_review(
6956 &mut self,
6957 why: &str,
6958 shell: &[String],
6959 worktree: &Path,
6960 ) -> Result<()> {
6961 self.stop_reviewing_inner(why, shell, worktree, true).await
6962 }
6963
6964 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6965 self.stop_reviewing_inner(why, shell, worktree, false).await
6966 }
6967
6968 async fn stop_reviewing_inner(
6969 &mut self,
6970 why: &str,
6971 shell: &[String],
6972 worktree: &Path,
6973 repairable: bool,
6974 ) -> Result<()> {
6975 let round_idx = self.state.reviews.len() - 1;
6976 let needs_catchup_run = matches!(
6984 self.state.reviews[round_idx].e2e_status(),
6985 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6986 );
6987 if needs_catchup_run {
6988 let round = self.state.reviews[round_idx].round;
6989 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6990 let commands = self.state.config.verify.e2e.clone();
6991 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6992 let cache_dir = self.state.config.cache_dir();
6993 let context = format!(
6994 "round {round}: verification unresolved, catching up before the final decision"
6995 );
6996 let wait = self.repository_wait();
6997 let (outcomes, verify_retried) = with_cache_lease(
6998 &mut self.state,
6999 cache_dir.as_deref(),
7000 "e2e",
7001 "e2e",
7002 worktree,
7003 &attempted_head,
7004 timeout,
7005 &context,
7006 &wait,
7007 |state, budget| {
7008 let shell = shell.to_vec();
7009 let commands = commands.clone();
7010 let context = context.clone();
7011 async move {
7012 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7013 .await
7014 }
7015 },
7016 )
7017 .await?;
7018 let last = &mut self.state.reviews[round_idx];
7019 last.e2e = outcomes;
7020 last.verify_retried = verify_retried;
7021 last.verified_head = Some(attempted_head);
7028 last.verified_at = Some(Timestamp::now());
7029 if verify_inconclusive(&last.e2e) {
7030 self.state.save()?;
7037 return Ok(());
7038 }
7039 last.e2e_deferred = false;
7040 }
7041 let last = &self.state.reviews[round_idx];
7042 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7043
7044 match last.e2e_status() {
7045 E2eStatus::Failed => {
7046 let red: Vec<String> = last
7047 .e2e
7048 .iter()
7049 .filter(|o| !o.ok())
7050 .map(|o| {
7051 format!(
7052 "`{}` -> {:?}\n{}",
7053 o.command,
7054 o.code,
7055 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7056 )
7057 })
7058 .collect();
7059 let can_repair = last.blocking == 0 && last.fix.is_none();
7060 self.state.reviews[round_idx].clean = false;
7061 self.state
7062 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7063 self.state.status = RunStatus::Blocked;
7064 if repairable
7065 && can_repair
7066 && self.state.reviews.len() < self.state.config.graph.review_rounds
7067 {
7068 self.state.status = RunStatus::Reviewing;
7069 }
7070 }
7071 E2eStatus::ResourceBlocked => {
7076 self.state.event(
7077 "review",
7078 format!(
7079 "{why}; e2e could not run (resource unavailable); not \
7080 deciding yet"
7081 ),
7082 );
7083 }
7084 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7085 self.state.event(
7086 "review",
7087 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7088 );
7089 self.record_contested_handoff();
7090 self.state.status = RunStatus::Gating;
7091 }
7092 }
7093 self.state.save()?;
7094 Ok(())
7095 }
7096
7097 async fn gate(&mut self) -> Result<()> {
7100 if self.state.status == RunStatus::Failed
7112 || self
7113 .state
7114 .base_sync
7115 .as_ref()
7116 .is_some_and(|s| s.conflict.is_some())
7117 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7118 != Some(RunStatus::Gating)
7119 {
7120 return Ok(());
7121 }
7122 if self.state.gate_ran {
7123 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7134 self.state.status = RunStatus::Blocked;
7135 self.state.save()?;
7136 }
7137 return Ok(());
7138 }
7139 let Some(winner) = self.state.winner().cloned() else {
7140 return Ok(());
7141 };
7142 self.state.status = RunStatus::Gating;
7143 let mut outcomes = self.run_gate(&winner).await?;
7144 loop {
7145 if verify_inconclusive(&outcomes) {
7156 self.state.save()?;
7157 return Ok(());
7158 }
7159 if outcomes.iter().all(CommandOutcome::ok) {
7160 break;
7161 }
7162 match self.gate_fix_round(&winner, &outcomes).await? {
7163 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7164 GateFix::Stop => break,
7165 GateFix::Defer => {
7166 self.state.save()?;
7167 return Ok(());
7168 }
7169 }
7170 }
7171 let passed = outcomes.iter().all(CommandOutcome::ok);
7172 self.state.gate = outcomes;
7173 self.state.gate_ran = true;
7174 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7175 if !passed {
7176 self.state.status = RunStatus::Blocked;
7177 let spent = self.state.gate_fixes.len();
7178 self.state.event(
7179 "gate",
7180 if spent == 0 {
7181 "gate failed; not merging".to_owned()
7182 } else {
7183 format!("gate failed after {spent} gate-fix round(s); not merging")
7184 },
7185 );
7186 }
7187 self.state.save()?;
7188 Ok(())
7189 }
7190
7191 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7201 let commands = state.config.verify.pre_gate.clone();
7202 if commands.is_empty() {
7203 return;
7204 }
7205 let shell = state.config.shell();
7206 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7207 let (outcomes, _) = run_commands(
7208 state,
7209 "pre_gate",
7210 "pre_gate",
7211 0,
7212 &shell,
7213 &commands,
7214 &winner.worktree,
7215 timeout,
7216 )
7217 .await;
7218 for o in &outcomes {
7219 if !o.ok() {
7220 tracing::warn!(
7221 "pre_gate `{}` failed ({:?}); the gate decides",
7222 o.command,
7223 o.code
7224 );
7225 }
7226 state.event(
7227 "pre_gate",
7228 format!(
7229 "`{}` -> {}",
7230 o.command,
7231 if o.ok() {
7232 "pass".to_owned()
7233 } else {
7234 format!(
7235 "FAIL ({:?})\n{}",
7236 o.code,
7237 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7238 )
7239 }
7240 ),
7241 );
7242 }
7243 state.pre_gate = outcomes;
7244 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7245 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7246 Ok(head) => {
7247 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7248 state.pre_gate_commit = Some(head);
7249 }
7250 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7251 },
7252 Ok(false) => {}
7253 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7254 }
7255 if let Err(e) = state.save() {
7256 tracing::warn!("could not persist the pre_gate record: {e:#}");
7257 }
7258 }
7259
7260 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7263 let wait = self.repository_wait();
7264 run_landing_gate(&mut self.state, winner, &wait).await
7265 }
7266
7267 async fn gate_fix_round(
7280 &mut self,
7281 winner: &Candidate,
7282 outcomes: &[CommandOutcome],
7283 ) -> Result<GateFix> {
7284 let cap = self.state.config.graph.gate_fix_rounds;
7285 let spent = self.state.gate_fixes.len();
7286 if spent >= cap {
7287 if cap > 0 {
7288 self.state.event(
7289 "gate",
7290 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7291 );
7292 }
7293 return Ok(GateFix::Stop);
7294 }
7295 if !gate_fixable(outcomes) {
7296 self.state.event(
7297 "gate",
7298 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7299 command or similar); not spending a fix round on it",
7300 );
7301 return Ok(GateFix::Stop);
7302 }
7303 let min_free = self.state.config.disk.min_free_bytes;
7304 if min_free > 0 {
7305 match crate::disk::free_bytes(&winner.worktree) {
7306 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7307 Ok(free) => {
7308 self.state.event(
7309 "gate",
7310 format!(
7311 "only {free} bytes free ({min_free} required by `[disk] \
7312 min_free_bytes`); not spending a fix round on a failure the disk \
7313 may explain"
7314 ),
7315 );
7316 return Ok(GateFix::Stop);
7317 }
7318 Err(e) => {
7319 self.state.event(
7320 "gate",
7321 format!("free disk space could not be measured ({e:#}); no fix round"),
7322 );
7323 return Ok(GateFix::Stop);
7324 }
7325 }
7326 }
7327
7328 let attempt = spent + 1;
7329 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7330 let base = self.landing_base();
7331 let fix_prompt = prompt::gate_fix_in(
7332 &self.state.instruction,
7333 &failed,
7334 attempt,
7335 cap,
7336 &self.state.config.graph.language,
7337 &self.state.config.graph.github_language,
7338 );
7339 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7340 let sessions = self.state.config.graph.sessions;
7341 let artifacts = self.merge_repair_artifacts();
7342 self.state.event(
7343 "gate",
7344 format!("gate failed; gate-fix round {attempt} of {cap}"),
7345 );
7346 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7347 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7348 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7351 head: before.clone(),
7352 passed: false,
7353 outcomes: Vec::new(),
7354 });
7355 self.state.save()?;
7356 let (_, seat, out) = self
7357 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7358 prompt: fix_prompt.clone(),
7359 spec,
7360 seat,
7361 cwd: winner.worktree.clone(),
7362 timeout,
7363 allow_write: true,
7364 sessions,
7365 artifacts: artifacts.clone(),
7366 stem: format!("gate-fix-{attempt}"),
7367 handover: None,
7368 })
7369 .await;
7370 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7373 return Ok(GateFix::Defer);
7374 }
7375 let mut record = GateFixRecord {
7376 agent: seat.agent.clone(),
7377 failed,
7378 notes: String::new(),
7379 committed: false,
7380 error: None,
7381 };
7382 match out {
7383 AgentOutcome::Ok(o) => {
7384 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7387 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7388 }
7389 }
7390 AgentOutcome::Dropped(_) => {
7391 record.error = Some("the CLI dropped the stream".to_owned());
7392 }
7393 AgentOutcome::Quota(o) => {
7394 self.state.quota.push(QuotaLoss {
7395 seat: seat.key.clone(),
7396 node: "gate-fix".to_owned(),
7397 at: Timestamp::now(),
7398 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7399 });
7400 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7401 }
7402 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7403 }
7404 self.state.seats.insert(seat.key.clone(), seat);
7405 let coordination = self.coordinate(&winner.worktree).await?;
7406 if let Ok(r) = git::rescue_commit(
7407 &winner.worktree,
7408 &format!("magi: gate fix {attempt} (uncommitted work)"),
7409 )
7410 .await
7411 {
7412 self.state.note_withheld("gate-fix", &r.withheld);
7413 }
7414 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7415 record.committed = after != before;
7416 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7417 let note = record.error.clone();
7418 self.state.gate_fixes.push(record);
7419 self.state.save()?;
7420 if !changed {
7421 self.state.event(
7422 "gate",
7423 match note {
7424 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7425 None => format!("gate-fix round {attempt}: the tree did not change"),
7426 },
7427 );
7428 return Ok(GateFix::Stop);
7429 }
7430 self.state.event(
7431 "gate",
7432 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7433 );
7434
7435 drop(coordination);
7436 let wait = self.repository_wait();
7437 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7438 }
7439
7440 async fn guarded_pr_message(
7447 &mut self,
7448 winner: &Candidate,
7449 facts: Option<&BranchFacts>,
7450 posting: bool,
7451 ) -> Result<Option<PrMessage>> {
7452 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7453 let head = match final_head.as_deref() {
7454 Some(h) => crate::pr_outcome::FinalHead::At(h),
7455 None => crate::pr_outcome::FinalHead::Unknown,
7456 };
7457 let summarized = if posting {
7459 Some(self.summarize_change(winner, final_head.as_deref()).await)
7460 } else {
7461 None
7462 };
7463 if self.state.resource_block.is_some() {
7468 self.state.event(
7469 "pr-summary",
7470 "a shared resource was busy; nothing was published, resume to summarize again",
7471 );
7472 return Ok(None);
7473 }
7474 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7475 anyhow::bail!(
7476 "the final commit moved during public description generation; retry before publishing"
7477 );
7478 }
7479 let note = match (&summarized, &self.state.change_summary) {
7480 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7481 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7482 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7483 (None, _) => ChangeNote::NotAsked,
7484 };
7485 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7486 if !posting {
7487 let identity = crate::scrub::Identity::current();
7488 return Ok(Some(PrMessage {
7489 title: crate::scrub::scrub(&pr.title, &identity),
7490 body: crate::scrub::scrub(&pr.body, &identity),
7491 }));
7492 }
7493 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7496 if let Some(known) = self
7497 .state
7498 .github_text
7499 .clone()
7500 .filter(|g| g.fingerprint == raw_fp)
7501 {
7502 return self.settle_withheld(winner, &pr, known).await;
7503 }
7504 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7505 let mut violations =
7506 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7507 let identity = crate::scrub::Identity::current();
7508 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7509 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7510 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7511 {
7512 violations.push(crate::github_text::Violation::SensitiveData);
7513 }
7514 let has_language = violations
7518 .iter()
7519 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7520 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7521 self.state.event(
7522 "github-text",
7523 if has_language {
7524 format!("description rejected: {violations:?}; requesting one rewrite")
7525 } else {
7526 format!("description rejected: {violations:?}; asking the owner")
7527 },
7528 );
7529 let mut rewritten = false;
7530 let mut rejected_rewrite = None;
7531 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7532 {
7533 let output = text;
7534 if let Some(title) = summary_title(&output) {
7535 let raw_body = summary_without_title(&output);
7538 let clean_title = crate::scrub::scrub(&title, &identity);
7539 let mut body = crate::scrub::scrub(&raw_body, &identity);
7540 let redacted = clean_title != title || body != raw_body;
7541 let title = clean_title;
7542 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7543 if !complete {
7544 self.state.event(
7545 "github-text",
7546 "description rewrite omitted required findings",
7547 );
7548 }
7549 if !body.trim().is_empty() {
7550 body.push_str("\n\n");
7551 body.push_str(&public_pr_evidence(&self.state, head));
7552 }
7553 let marker = format!("magi:run/{}", self.state.id);
7554 if !body.trim().is_empty() && !body.contains(&marker) {
7555 body.push_str(&format!("\n\n{marker}"));
7556 }
7557 let rewrite = PrMessage {
7558 title: title.clone(),
7559 body: body.clone(),
7560 };
7561 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7562 let mut failed = crate::github_text::check_for(
7563 &self.state.config.graph,
7564 &title,
7565 &body,
7566 redecision,
7567 );
7568 if (!crate::github_text::shareable(&title)
7569 || !crate::github_text::shareable(&body))
7570 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7571 {
7572 failed.push(crate::github_text::Violation::SensitiveData);
7573 }
7574 if complete
7575 && !title.trim().is_empty()
7576 && !body.trim().is_empty()
7577 && !failed.is_empty()
7578 {
7579 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7580 title: title.clone(),
7581 body: body.clone(),
7582 failed: failed
7583 .iter()
7584 .map(|v| crate::github_text::category(*v).to_owned())
7585 .collect(),
7586 });
7587 }
7588 if complete
7589 && !title.trim().is_empty()
7590 && !raw_body.trim().is_empty()
7591 && failed.is_empty()
7592 {
7593 decision = redecision;
7594 pr = PrMessage { title, body };
7595 rewritten = true;
7596 self.state
7597 .event("github-text", "description rewrite passed");
7598 if redacted {
7599 self.state.event(
7600 "github-text",
7601 "sensitive data removed from description rewrite",
7602 );
7603 }
7604 }
7605 }
7606 }
7607 if !rewritten {
7608 self.state.event(
7609 "github-text",
7610 "description rewrite unavailable or rejected; asking the owner",
7611 );
7612 if let Some(w) =
7613 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7614 {
7615 self.retire_github_text_question("the withheld text changed");
7618 let record = crate::run::GithubTextAsk {
7619 rewrite: rejected_rewrite,
7620 fingerprint: raw_fp,
7621 title: w.title,
7622 body: w.body,
7623 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7624 question: None,
7625 asks: 0,
7626 resolved: false,
7627 chosen_title: None,
7628 choice: None,
7629 };
7630 return self.settle_withheld(winner, &pr, record).await;
7631 }
7632 }
7633 }
7634 let (title, body) =
7635 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7636 Ok(Some(PrMessage { title, body }))
7637 }
7638
7639 async fn summarize_change(
7650 &mut self,
7651 winner: &Candidate,
7652 head: Option<&str>,
7653 ) -> std::result::Result<(), String> {
7654 let Some(head) = head else {
7655 return Err("the final commit could not be read".to_owned());
7656 };
7657 let remote = self.state.config.merge.remote.clone();
7658 let tracking = format!("{remote}/{}", self.state.base_branch);
7659 if !git::rev_exists(&winner.worktree, &tracking).await {
7660 return Err(format!("the remote base `{tracking}` could not be read"));
7661 }
7662 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7663 Ok(b) if !b.is_empty() => b,
7664 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7665 };
7666 let evidence = crate::pr_outcome::summary_evidence(
7667 &self.state,
7668 crate::pr_outcome::FinalHead::At(head),
7669 );
7670 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7671 if self
7672 .state
7673 .change_summary
7674 .as_ref()
7675 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7676 {
7677 return Ok(());
7678 }
7679 if self.state.change_summary.take().is_some() {
7680 self.state.event(
7681 "pr-summary",
7682 "the final diff or evidence changed; the earlier change summary was discarded",
7683 );
7684 }
7685 if let Err(e) = write_artifact(
7687 &self.state,
7688 &format!("public-description-{fingerprint}.md"),
7689 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7690 ) {
7691 tracing::warn!("could not save public description evidence: {e:#}");
7692 }
7693 let chain = match agent::pick_chain(
7694 &self.state.config.agents,
7695 self.state.config.roles.synthesizer.as_ref(),
7696 &agent::installed,
7697 "synthesizer",
7698 ) {
7699 Ok(chain) => chain,
7700 Err(e) => {
7701 tracing::warn!("no agent for the change summary: {e:#}");
7702 self.state.event("pr-summary", "no summary agent available");
7703 return Err("no agent was available to summarize it".to_owned());
7704 }
7705 };
7706 let subjects = git::subjects(&winner.worktree, &base, head)
7707 .await
7708 .unwrap_or_default();
7709 let stat = git::diff_stat(&winner.worktree, &base, head)
7710 .await
7711 .unwrap_or_default();
7712 let stat: String = stat.chars().take(6_000).collect();
7713 let mut prompt = crate::prompt::change_summary_in(
7714 &base,
7715 head,
7716 &subjects,
7717 stat.trim(),
7718 &self.state.config.graph.github_language,
7719 );
7720 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7721 let sessions = self.state.config.graph.sessions;
7722 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7723 let prompts = self.state.config.prompts.clone();
7724 let cache = self.state.config.cache_dir();
7725 let run = self.state.id.clone();
7726 for (i, spec) in chain.into_iter().enumerate() {
7727 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7728 let github_spec = match agent::for_github_text(
7729 &spec,
7730 &self.state.config.graph.github_language,
7731 &winner.worktree,
7732 ) {
7733 Ok(spec) => spec,
7734 Err(e) => {
7735 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7736 continue;
7737 }
7738 };
7739 let job = SeatJob {
7740 spec: github_spec,
7741 seat,
7742 cwd: winner.worktree.clone(),
7743 prompt: prompt.clone(),
7744 timeout,
7745 allow_write: false,
7746 sessions,
7747 artifacts: agent::artifacts_dir(&self.state.dir()),
7748 stem: if i == 0 {
7749 "change-summary".to_owned()
7750 } else {
7751 format!("change-summary-{}", spec.id)
7752 },
7753 handover: None,
7754 };
7755 let ctx = WaveCtx {
7756 run: &run,
7757 node: "pr-summary",
7758 prompts: &prompts,
7759 cache: cache.as_deref(),
7760 round: None,
7761 carry_seats: false,
7762 pauses: Some((&self.pause, &self.interrupt)),
7763 };
7764 let (seat, outcome) =
7765 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7766 self.state.seats.insert(seat.key.clone(), seat);
7767 let text = match outcome {
7768 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7769 AgentOutcome::ResourceBlocked(why) => {
7770 if self.state.resource_block.is_none() {
7773 self.state.resource_block = Some(crate::run::ResourceBlock {
7774 cause: crate::run::ResourceBlockCause::Other,
7775 node: "pr-summary".to_owned(),
7776 kind: "shared resource".to_owned(),
7777 detail: why,
7778 at: Timestamp::now(),
7779 });
7780 }
7781 return Err("a shared resource was busy".to_owned());
7782 }
7783 _ => continue,
7784 };
7785 let blind = &self.state.config.blind;
7786 let Some(title) = summary_title(&text) else {
7787 continue;
7788 };
7789 let body: String = summary_without_title(&text)
7790 .lines()
7791 .filter(|l| !l.contains("magi:run/"))
7792 .collect::<Vec<_>>()
7793 .join("\n");
7794 let body = blind::sanitize_prose(body.trim(), blind);
7795 let title = blind::sanitize_prose(&title, blind);
7796 if body.trim().is_empty() || title.trim().is_empty() {
7797 continue;
7798 }
7799 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7800 continue;
7801 }
7802 if git::rev_parse(&winner.worktree, "HEAD")
7804 .await
7805 .ok()
7806 .as_deref()
7807 != Some(head)
7808 {
7809 return Err("the branch moved while the summary was written".to_owned());
7810 }
7811 if git::merge_base(&winner.worktree, &tracking, head)
7812 .await
7813 .ok()
7814 .as_deref()
7815 != Some(base.as_str())
7816 || crate::pr_outcome::summary_evidence(
7817 &self.state,
7818 crate::pr_outcome::FinalHead::At(head),
7819 ) != evidence
7820 {
7821 return Err("the summary evidence changed while it was written".to_owned());
7822 }
7823 self.state.change_summary = Some(crate::run::ChangeSummary {
7824 head: head.to_owned(),
7825 base,
7826 evidence: fingerprint,
7827 title,
7828 body,
7829 });
7830 self.state.event(
7831 "pr-summary",
7832 format!("change summary written by {} for the final diff", spec.id),
7833 );
7834 if let Err(e) = self.state.save() {
7835 tracing::warn!("could not save the change summary: {e:#}");
7836 }
7837 return Ok(());
7838 }
7839 self.state
7840 .event("pr-summary", "no agent produced a change summary");
7841 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7842 }
7843
7844 async fn rewrite_pr_text(
7854 &mut self,
7855 winner: &Candidate,
7856 pr: &PrMessage,
7857 violations: &[crate::github_text::Violation],
7858 ) -> Option<String> {
7859 let prompt = format!(
7860 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7861 pr.title, pr.body
7862 );
7863 let sessions = self.state.config.graph.sessions;
7864 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7865 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7868 let key = format!("impl-{}", winner.label);
7869 if let Some(spec) = self
7870 .state
7871 .config
7872 .agents
7873 .iter()
7874 .find(|a| a.id == winner.agent)
7875 .cloned()
7876 && let Some(existing) = self.state.seats.get(&key)
7877 && existing.agent == spec.id
7878 && agent::has_session(spec.kind, existing, sessions)
7879 {
7880 attempts.push((spec, Some(existing.clone())));
7881 }
7882 match agent::pick_chain(
7883 &self.state.config.agents,
7884 self.state.config.roles.language_judge.as_ref(),
7885 &agent::installed,
7886 "language_judge",
7887 ) {
7888 Ok(chain) => {
7889 for spec in chain {
7890 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7891 continue;
7892 }
7893 attempts.push((spec, None));
7894 }
7895 }
7896 Err(e) if attempts.is_empty() => {
7897 tracing::warn!("no agent for the description rewrite: {e:#}");
7898 self.state
7899 .event("github-text", "no rewrite agent available");
7900 return None;
7901 }
7902 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7903 }
7904 let prompts = self.state.config.prompts.clone();
7905 let cache = self.state.config.cache_dir();
7906 let run = self.state.id.clone();
7907 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7908 let resumed = resumed_seat.is_some();
7909 let seat = resumed_seat.unwrap_or_else(|| {
7910 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7911 });
7912 let github_spec = match agent::for_github_text(
7913 &spec,
7914 &self.state.config.graph.github_language,
7915 &winner.worktree,
7916 ) {
7917 Ok(spec) => spec,
7918 Err(e) => {
7919 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7920 continue;
7921 }
7922 };
7923 let job = SeatJob {
7924 spec: github_spec,
7925 seat,
7926 cwd: winner.worktree.clone(),
7927 prompt: prompt.clone(),
7928 timeout: retry_budget(timeout_full, resumed),
7929 allow_write: false,
7930 sessions,
7931 artifacts: agent::artifacts_dir(&self.state.dir()),
7932 stem: if i == 0 {
7933 "github-text-rewrite".to_owned()
7934 } else {
7935 format!("github-text-rewrite-{}", spec.id)
7936 },
7937 handover: None,
7938 };
7939 let ctx = WaveCtx {
7940 run: &run,
7941 node: "github-text",
7942 prompts: &prompts,
7943 cache: cache.as_deref(),
7944 round: None,
7945 carry_seats: false,
7946 pauses: Some((&self.pause, &self.interrupt)),
7947 };
7948 let (seat, outcome) =
7949 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7950 self.state.seats.insert(seat.key.clone(), seat);
7951 match outcome {
7952 AgentOutcome::Ok(o)
7953 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7954 {
7955 return Some(o.text);
7956 }
7957 AgentOutcome::ResourceBlocked(_) => return None,
7958 _ => {}
7959 }
7960 }
7961 None
7962 }
7963
7964 fn retire_github_text_question(&mut self, why: &str) {
7966 let Some(id) = self
7967 .state
7968 .github_text
7969 .as_ref()
7970 .and_then(|g| g.question.clone())
7971 else {
7972 return;
7973 };
7974 let _ = ask::Questions::open().update(&id, |q| {
7975 q.abandon(why);
7976 Ok(())
7977 });
7978 }
7979
7980 async fn settle_withheld(
7984 &mut self,
7985 winner: &Candidate,
7986 pr: &PrMessage,
7987 mut g: crate::run::GithubTextAsk,
7988 ) -> Result<Option<PrMessage>> {
7989 use crate::github_text as gt;
7990 let store = ask::Questions::open();
7991 let timeout = self.state.config.graph.answer_timeout;
7992 if !g.resolved {
7993 let mut standing = g
7994 .question
7995 .as_ref()
7996 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7997 if let Some(q) = standing.as_ref()
7998 && q.status.open()
7999 && gt::expired(q, timeout)
8000 {
8001 let why = format!("no answer within {}s of asking", timeout.max(1));
8002 standing = store
8003 .update(&q.id, |q| {
8004 q.abandon(&why);
8005 Ok(())
8006 })
8007 .ok()
8008 .map(|(q, ())| q);
8009 }
8010 let mut ask_again: Option<Option<Vec<String>>> = None;
8011 match standing {
8012 None if g.asks <= 1 => {
8015 g.asks = 0;
8016 ask_again = Some(None);
8017 }
8018 None => {
8019 g.resolved = true;
8020 g.chosen_title = None;
8021 g.choice = Some(GithubTextChoice::Fallback);
8022 }
8023 Some(q) if q.status.open() => {
8024 self.park_on_github_text(&g)?;
8025 return Ok(None);
8026 }
8027 Some(q) => match gt::read_reply(&q) {
8028 gt::Reply::Fallback => {
8029 g.resolved = true;
8030 g.chosen_title = None;
8031 g.choice = Some(GithubTextChoice::Fallback);
8032 }
8033 gt::Reply::Original => {
8034 g.resolved = true;
8035 g.chosen_title = None;
8036 g.choice = Some(GithubTextChoice::Original);
8037 }
8038 gt::Reply::Rewrite => {
8039 g.resolved = true;
8040 g.chosen_title = None;
8041 g.choice = Some(if g.rewrite.is_some() {
8042 GithubTextChoice::Rewrite
8043 } else {
8044 GithubTextChoice::Fallback
8045 });
8046 }
8047 gt::Reply::Title(text) => {
8048 let first = text
8049 .lines()
8050 .map(str::trim)
8051 .find(|l| !l.is_empty())
8052 .unwrap_or("");
8053 let decision = if first.is_empty() {
8054 None
8055 } else {
8056 let candidate = PrMessage {
8057 title: first.to_owned(),
8058 body: String::new(),
8059 };
8060 self.judge_pr_language(&winner.worktree, &candidate).await
8061 };
8062 match gt::vet_title(
8063 &text,
8064 decision,
8065 self.state.config.graph.github_english(),
8066 ) {
8067 Ok(title) => {
8068 g.resolved = true;
8069 g.chosen_title = Some(title);
8070 g.choice = Some(GithubTextChoice::MyTitle);
8071 }
8072 Err(cats) if g.asks < 2 => {
8073 ask_again = Some(Some(
8074 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8075 ))
8076 }
8077 Err(_) => {
8078 g.resolved = true;
8079 g.chosen_title = None;
8080 g.choice = Some(GithubTextChoice::Fallback);
8081 }
8082 }
8083 }
8084 },
8085 }
8086 if let Some(rejected) = ask_again {
8087 let w = gt::Withheld {
8090 title: g.title,
8091 body: g.body,
8092 categories: g.categories.clone(),
8093 };
8094 let artifact = match crate::run::write_artifact(
8097 &self.state,
8098 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8099 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8100 ) {
8101 Ok(p) => Some(p),
8102 Err(e) => {
8103 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8104 None
8105 }
8106 };
8107 let (detail, comparison) = gt::question_content(
8108 &w,
8109 &pr.title,
8110 &pr.body,
8111 &self.state.id,
8112 rejected.as_deref(),
8113 artifact.as_deref(),
8114 g.rewrite.as_ref(),
8115 );
8116 let mut q = ask::Question::new(
8117 self.state.id.clone(),
8118 gt::ASK_NODE.to_owned(),
8119 gt::ASK_SEAT.to_owned(),
8120 gt::question_summary(&self.state.config.graph.language, &w),
8121 detail,
8122 gt::question_choices(&w, g.rewrite.as_ref()),
8123 );
8124 q.posting_comparison = Some(comparison);
8125 q.answer_timeout = timeout;
8126 g.question = Some(q.id.clone());
8129 g.asks += 1;
8130 self.state.github_text = Some(g.clone());
8131 self.state.event(
8132 "github-text",
8133 format!("asking the owner about the withheld text ({})", q.short()),
8134 );
8135 self.state.save()?;
8136 store
8137 .put_executable(&mut q)
8138 .context("file the posting-gate question")?;
8139 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8140 tracing::warn!(
8141 "could not notify about posting-gate question {}: {e:#}",
8142 q.short()
8143 );
8144 }
8145 self.park_on_github_text(&g)?;
8146 return Ok(None);
8147 }
8148 }
8149 let resolved = g.chosen_title.clone();
8150 let original = g.choice == Some(GithubTextChoice::Original);
8151 self.state.event(
8153 "github-text",
8154 if original {
8155 format!(
8156 "posting the original text for: {}",
8157 match (g.title, g.body) {
8158 (true, true) => "title, description",
8159 (true, false) => "title",
8160 _ => "description",
8161 }
8162 )
8163 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8164 "posting the saved rewrite title and description".to_owned()
8165 } else if resolved.is_some() {
8166 "posting the owner's replacement title".to_owned()
8167 } else {
8168 "posting the neutral text for the withheld fields".to_owned()
8169 },
8170 );
8171 self.state.github_text = Some(g.clone());
8172 self.state.save()?;
8173 if g.choice == Some(GithubTextChoice::Rewrite)
8174 && let Some(r) = g.rewrite.as_ref()
8175 {
8176 return Ok(Some(PrMessage {
8177 title: r.title.clone(),
8178 body: r.body.clone(),
8179 }));
8180 }
8181 let w = gt::Withheld {
8182 title: g.title,
8183 body: g.body,
8184 categories: g.categories.clone(),
8185 };
8186 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8187 if let Some(t) = resolved {
8188 title = t;
8189 }
8190 let english = gt::LanguageDecision {
8192 title_english: true,
8193 body_english: true,
8194 body_complete: true,
8195 };
8196 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8197 if original {
8198 if g.title {
8201 title = pr.title.clone();
8202 }
8203 if g.body {
8204 body = pr.body.clone();
8205 }
8206 }
8207 Ok(Some(PrMessage { title, body }))
8208 }
8209
8210 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8211 self.state.github_text = Some(g.clone());
8212 self.state.end_pass_parked(ParkCause::Question);
8213 self.state.parked = true;
8214 self.state.event(
8215 "github-text",
8216 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8217 );
8218 self.state.save()
8219 }
8220
8221 async fn judge_pr_language(
8224 &mut self,
8225 cwd: &Path,
8226 pr: &PrMessage,
8227 ) -> Option<crate::github_text::LanguageDecision> {
8228 self.state.config.roles.language_judge.as_ref()?;
8229 if !self.state.config.graph.language_gate() {
8230 return None;
8231 }
8232 let decision =
8233 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8234 self.state.event(
8235 "github-text",
8236 if decision.is_some() {
8237 "language decided by the language judge"
8238 } else {
8239 "language judge unavailable; using built-in heuristics"
8240 },
8241 );
8242 decision
8243 }
8244
8245 async fn merge(&mut self) -> Result<()> {
8246 if self
8261 .state
8262 .base_sync
8263 .as_ref()
8264 .is_some_and(|s| s.conflict.is_some())
8265 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8266 != Some(RunStatus::Gating)
8267 || !self.state.gate_allows_progress()
8275 {
8276 return Ok(());
8277 }
8278 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8287 return Ok(());
8288 }
8289 let Some(winner) = self.state.winner().cloned() else {
8290 return Ok(());
8291 };
8292 let repo = self.state.repo.clone();
8293 let base = self.state.base_branch.clone();
8294 let mode = self.state.config.merge.mode;
8295 let style = self.state.config.merge.style;
8296 let facts = if is_review_run(&self.state) {
8297 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8298 let start = review_base(
8299 &repo,
8300 &self.state.config.merge.remote,
8301 &base,
8302 &self.state.base_commit,
8303 &winner.branch,
8304 )
8305 .await;
8306 branch_facts(&repo, &start, &winner.branch).await
8307 } else {
8308 None
8309 };
8310 let leaked = if is_review_run(&self.state) {
8313 leaked_subjects(&self.state, &winner.branch).await
8314 } else {
8315 Some(Vec::new())
8316 };
8317 let Some(pr) = self
8321 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8322 .await?
8323 else {
8324 return Ok(());
8325 };
8326 let message = pr.commit_message();
8327
8328 let outcome = match mode {
8329 MergeMode::None => MergeOutcome {
8330 mode,
8331 ok: true,
8332 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8333 empty: false,
8334 },
8335 MergeMode::Pr | MergeMode::Local
8336 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8337 {
8338 MergeOutcome {
8339 mode,
8340 ok: false,
8341 detail: empty_candidate_detail(&self.state, &base),
8342 empty: true,
8343 }
8344 }
8345 MergeMode::Local => {
8346 let on = git::current_branch(&repo).await?;
8347 if on.as_deref() != Some(base.as_str()) {
8348 MergeOutcome {
8349 mode,
8350 ok: false,
8351 detail: format!(
8352 "{} has {} checked out, not the base branch {base}",
8353 repo.display(),
8354 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8355 ),
8356 empty: false,
8357 }
8358 } else if !git::is_clean(&repo).await? {
8359 MergeOutcome {
8360 mode,
8361 ok: false,
8362 detail: format!("{} is dirty; refusing to merge", repo.display()),
8363 empty: false,
8364 }
8365 } else {
8366 let out = match style {
8367 MergeStyle::Merge => {
8368 git::merge_no_ff(&repo, &winner.branch, &message).await?
8369 }
8370 MergeStyle::Squash => {
8371 git::merge_squash(&repo, &winner.branch, &message).await?
8372 }
8373 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8374 };
8375 MergeOutcome {
8376 mode,
8377 ok: out.ok(),
8378 detail: if out.ok() { out.stdout } else { out.stderr },
8379 empty: false,
8380 }
8381 }
8382 }
8383 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8384 self.state.pr_publication_pending = true;
8385 self.state.save()?;
8386 let out =
8387 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8388 self.state.pr_publication_pending = out.is_err();
8389 MergeOutcome {
8390 mode,
8391 ok: out.is_ok(),
8392 detail: match out {
8393 Ok(url) => url,
8394 Err(e) => format!("{e:#}"),
8395 },
8396 empty: false,
8397 }
8398 }
8399 MergeMode::Pr => {
8400 let remote = self.state.config.merge.remote.clone();
8401 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8402 if !pushed.ok() {
8403 MergeOutcome {
8404 mode,
8405 ok: false,
8406 detail: pushed.stderr,
8407 empty: false,
8408 }
8409 } else {
8410 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8416 let out = match pr_merge_plan(found) {
8417 PrPlan::Create => {
8418 gh_pr_create(
8419 &winner.worktree,
8420 &base,
8421 &winner.branch,
8422 &pr.title,
8423 &pr.body,
8424 )
8425 .await
8426 }
8427 PrPlan::Adopt { url, title } => {
8428 self.state
8429 .event("merge", format!("Pr: adopted open pull request {url}"));
8430 if self.state.github_text.as_ref().is_some_and(|g| {
8431 g.resolved
8432 && g.choice == Some(GithubTextChoice::Rewrite)
8433 && g.rewrite.is_some()
8434 }) {
8435 land::set_pr_text_verbatim(
8438 &winner.worktree,
8439 &url,
8440 &pr.title,
8441 &pr.body,
8442 )
8443 .await
8444 .map(|()| url)
8445 } else {
8446 if title != pr.title
8447 && (!is_review_run(&self.state)
8448 || leaked
8449 .as_deref()
8450 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8451 && let Err(e) =
8452 if self.state.github_text.as_ref().is_some_and(|g| {
8453 g.resolved
8454 && g.title
8455 && g.choice == Some(GithubTextChoice::Original)
8456 }) {
8457 land::set_pr_title_verbatim(
8459 &winner.worktree,
8460 &url,
8461 &pr.title,
8462 )
8463 .await
8464 } else {
8465 land::set_pr_title(
8466 &mut self.state,
8467 &winner.worktree,
8468 &url,
8469 &pr.title,
8470 )
8471 .await
8472 }
8473 {
8474 tracing::warn!("could not refresh title of {url}: {e:#}");
8475 self.state
8476 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8477 }
8478 Ok(url)
8479 }
8480 }
8481 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8482 };
8483 match out {
8484 Ok(url) => MergeOutcome {
8485 mode,
8486 ok: true,
8487 detail: url,
8488 empty: false,
8489 },
8490 Err(e) => MergeOutcome {
8491 mode,
8492 ok: false,
8493 detail: e.to_string(),
8494 empty: false,
8495 },
8496 }
8497 }
8498 }
8499 };
8500
8501 self.state.status = match (mode, outcome.ok) {
8502 (MergeMode::None, _) => RunStatus::Ready,
8503 (_, true) if outcome.empty => RunStatus::Ready,
8504 (MergeMode::Pr, true) => RunStatus::Ready,
8505 (_, true) => RunStatus::Merged,
8506 (_, false) => RunStatus::Blocked,
8507 };
8508 self.state.event(
8509 "merge",
8510 format!(
8511 "{:?}: {}",
8512 mode,
8513 outcome.detail.lines().next().unwrap_or("")
8514 ),
8515 );
8516 self.state.merge = Some(outcome);
8517 self.state.save()?;
8518
8519 if self.state.config.graph.land
8525 && mode == MergeMode::Pr
8526 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8527 {
8528 self.run_land().await?;
8529 }
8530 self.settle_questions();
8535 Ok(())
8536 }
8537
8538 async fn run_land(&mut self) -> Result<()> {
8549 crate::pr_head::ensure_owner(&self.state)?;
8550 let url = self
8551 .state
8552 .pr
8553 .as_ref()
8554 .map(|pr| pr.url.clone())
8555 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8556 .unwrap_or_default();
8557 let url = url.lines().next().unwrap_or("").trim().to_owned();
8558 if !url.starts_with("http") {
8559 return Ok(());
8560 }
8561 loop {
8564 match land::land(&mut self.state, &url).await {
8565 Ok(_)
8566 if self
8567 .state
8568 .merge_repairs
8569 .last()
8570 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8571 {
8572 if self.repair_merge_review().await? {
8573 continue;
8574 }
8575 }
8576 Ok(pr) if self.state.parked => {
8577 let _ = pr;
8581 }
8582 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8583 self.state.status = RunStatus::Superseded;
8584 self.state.save()?;
8585 }
8586 Ok(pr) => {
8587 self.state.status = match pr.state {
8588 land::PrLifecycle::Merged => RunStatus::Merged,
8589 _ => RunStatus::Blocked,
8590 };
8591 if bump::should_release_bump(self.state.status)
8598 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8599 {
8600 self.state
8606 .event("bump", format!("release bump skipped: {e:#}"));
8607 }
8608 if self.state.status == RunStatus::Merged {
8611 crate::followup::after_merge(&mut self.state, &pr.url).await;
8612 }
8613 self.state.save()?;
8614 }
8615 Err(e) => {
8616 if e.is::<crate::repo_lock::ResourceBlocked>() {
8617 return Err(e);
8618 }
8619 self.state.status = RunStatus::Blocked;
8620 self.state.event("land", format!("gave up: {e}"));
8621 self.state.save()?;
8622 }
8623 }
8624 break;
8625 }
8626 Ok(())
8627 }
8628
8629 async fn repair_merge_review(&mut self) -> Result<bool> {
8632 use crate::run::MergeRepairPhase as Phase;
8633 crate::pr_head::ensure_owner(&self.state)?;
8634 let i = self.state.merge_repairs.len() - 1;
8635 let request = self.state.merge_repairs[i].clone();
8636 if request.phase == Phase::Fixing {
8637 let restored = match self.state.winner().cloned() {
8638 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8639 None => Err(anyhow::anyhow!("repair has no winning branch")),
8640 };
8641 let why = match restored {
8642 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8643 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8644 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8645 };
8646 self.hold_merge_repair(i, why)?;
8647 return Ok(false);
8648 }
8649 let failure = if i >= self.state.config.graph.land_rounds
8650 || self.state.config.graph.review_rounds == 0
8651 {
8652 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8653 } else {
8654 None
8655 };
8656 if let Some(why) = failure {
8657 self.hold_merge_repair(i, why)?;
8658 return Ok(false);
8659 }
8660 if request.phase == Phase::Requested {
8661 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8662 Ok(pr) => pr,
8663 Err(e) => {
8664 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8665 return Ok(false);
8666 }
8667 };
8668 self.state.merge_repairs[i].phase = Phase::Fixing;
8669 self.state.land_approval = None;
8670 self.state.land_armed_head = None;
8671 self.state.save()?;
8672 let evidence = serde_json::to_string_pretty(&request)?;
8675 let reason = format!(
8676 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8677 );
8678 let budget = self.state.config.graph.land_rounds;
8679 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8680 match result {
8681 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8682 Ok(land::Fixed::Failed(why)) => {
8683 self.hold_merge_repair(
8684 i,
8685 format!(
8686 "repair failed: {why}; inspect the branch and run an explicit review"
8687 ),
8688 )?;
8689 return Ok(false);
8690 }
8691 Err(e) => {
8692 self.hold_merge_repair(
8693 i,
8694 format!(
8695 "repair failed: {e:#}; inspect the branch and run an explicit review"
8696 ),
8697 )?;
8698 return Ok(false);
8699 }
8700 }
8701 self.begin_merge_repair_review(i)?;
8705 }
8706 self.review_loop().await?;
8707 self.gate().await?;
8708 if self.resource_halt()? || self.state.parked {
8709 return Ok(false);
8710 }
8711 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8712 != Some(RunStatus::Gating)
8713 || !self.state.gate_allows_progress()
8714 {
8715 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8716 return Ok(false);
8717 }
8718 let winner = self
8721 .state
8722 .winner()
8723 .cloned()
8724 .context("repair has no winning branch")?;
8725 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8726 if self.state.gate_head.as_deref() != Some(after.as_str())
8727 || !git::is_clean(&winner.worktree).await?
8728 {
8729 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8730 return Ok(false);
8731 }
8732 let expected = self
8733 .state
8734 .pr_pushed_head
8735 .as_deref()
8736 .unwrap_or(&request.head)
8737 .to_owned();
8738 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8739 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8742 let refspec = format!("{after}:refs/heads/{public}");
8743 let push = git::git_raw(
8744 &winner.worktree,
8745 &["push", &lease, &self.state.config.merge.remote, &refspec],
8746 )
8747 .await;
8748 match push {
8749 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8750 Ok(out) => {
8751 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8752 return Ok(false);
8753 }
8754 Err(e) => {
8755 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8756 return Ok(false);
8757 }
8758 }
8759 self.state.merge_repairs[i].phase = Phase::Complete;
8760 self.state.status = RunStatus::Landing;
8761 self.state.event(
8762 "repair",
8763 "new review and gate completed; fresh owner merge decision required",
8764 );
8765 self.state.save()?;
8766 Ok(true)
8767 }
8768
8769 fn merge_repair_artifacts(&self) -> PathBuf {
8770 let artifacts = agent::artifacts_dir(&self.state.dir());
8771 match self
8772 .state
8773 .merge_repairs
8774 .last()
8775 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8776 {
8777 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8778 None => artifacts,
8779 }
8780 }
8781
8782 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8783 self.state.reviews.clear();
8784 self.state.gate.clear();
8785 self.state.gate_ran = false;
8786 self.state.gate_head = None;
8787 self.state.gate_fixes.clear();
8788 self.state.gate_repair_e2e = None;
8789 self.state.pre_gate.clear();
8790 self.state.pre_gate_commit = None;
8791 self.state.contested_handoff = None;
8792 self.state.land_reruns = None;
8793 self.state.land_approval = None;
8794 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8795 self.state.status = RunStatus::Reviewing;
8796 self.state.save()
8797 }
8798
8799 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8800 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8801 self.state.merge_repairs[index].failure = Some(why.clone());
8802 self.state.status = RunStatus::Blocked;
8803 self.state.event("repair", why);
8804 self.state.save()
8805 }
8806
8807 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8811 if let Some(existing) = self.state.seats.get(key)
8812 && existing.agent == agent
8813 {
8814 return existing.clone();
8815 }
8816 let fresh = if self.state.seats.contains_key(key) {
8819 handover_seat(key, agent, self.state.next_seat_seed())
8820 } else {
8821 SeatState::new(key, agent, self.state.seed)
8822 };
8823 self.state.seats.insert(key.to_owned(), fresh.clone());
8824 fresh
8825 }
8826
8827 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8833 match self.state.seats.get(key) {
8834 Some(s) if s.agent != spec.id => {
8835 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8836 }
8837 _ => spec,
8838 }
8839 }
8840
8841 fn view(&self, c: &Candidate) -> CandidateView {
8843 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8844 .unwrap_or_default();
8845 let (patch, _) = blind::sanitize_patch(
8846 &format!("candidate {} patch", c.label),
8847 &raw,
8848 &self.state.config.blind,
8849 );
8850 CandidateView {
8851 label: c.label,
8852 branch: c.branch.clone(),
8853 summary: c.summary.clone(),
8854 stat: c.stat.clone(),
8855 patch,
8856 }
8857 }
8858
8859 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8861 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8862 prompt::judge(
8863 "(see above)",
8864 &views,
8865 self.roles.judges.len(),
8866 base_short,
8867 "en",
8868 )
8869 }
8870
8871 fn vote_prompt_full(
8877 &self,
8878 j: usize,
8879 labels: &[char],
8880 language: &str,
8881 candidates: &[Candidate],
8882 base_short: &str,
8883 ) -> String {
8884 let mut text = format!(
8885 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8886 prompt::final_vote(labels, language),
8887 self.state.instruction,
8888 self.candidate_block(candidates, base_short)
8889 );
8890 if let Some(own) = self
8891 .state
8892 .judgements
8893 .get(j)
8894 .filter(|r| !r.ranking.is_empty())
8895 {
8896 let reasons = own
8897 .reasons
8898 .iter()
8899 .map(|(k, v)| format!("- {k}: {v}"))
8900 .collect::<Vec<_>>()
8901 .join("\n");
8902 text.push_str(&format!(
8903 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8904 own.ranking.iter().collect::<String>(),
8905 if reasons.is_empty() {
8906 ""
8907 } else {
8908 ", because:\n"
8909 }
8910 ));
8911 }
8912 if !self.state.deliberation.is_empty() {
8913 text.push_str("\n# What was argued before this vote\n");
8914 for t in self.transcript(&[], j) {
8915 text.push_str(&format!(
8916 "\n## {}{}\n\n{}\n",
8917 t.who,
8918 if t.is_self { " (you)" } else { "" },
8919 t.body.trim()
8920 ));
8921 }
8922 }
8923 text
8924 }
8925
8926 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8933 let mut turns = Vec::new();
8934 for j in &self.state.judgements {
8935 if j.ranking.is_empty() {
8936 continue;
8937 }
8938 let reasons = j
8939 .reasons
8940 .iter()
8941 .map(|(k, v)| format!("- {k}: {v}"))
8942 .collect::<Vec<_>>()
8943 .join("\n");
8944 turns.push(Turn {
8945 who: format!("Judge {} (opening ranking)", j.judge),
8946 is_self: j.judge == self_idx + 1,
8947 body: format!(
8948 "Ranked {}{}{reasons}",
8949 j.ranking.iter().collect::<String>(),
8950 if reasons.is_empty() {
8951 ""
8952 } else {
8953 ", because:\n"
8954 }
8955 ),
8956 });
8957 }
8958 for t in self
8959 .state
8960 .deliberation
8961 .iter()
8962 .flat_map(|r| r.turns.iter())
8963 .chain(current)
8964 {
8965 turns.push(Turn {
8966 who: format!("Judge {}", t.judge),
8967 is_self: t.judge == self_idx + 1,
8968 body: t.body.clone(),
8969 });
8970 }
8971 turns
8972 }
8973}
8974
8975fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8977 agent::has_session(spec.kind, seat, sessions)
8978}
8979
8980fn next_untried_in_roster<'a>(
9001 roster: &'a [AgentSpec],
9002 start: usize,
9003 tried: &BTreeSet<String>,
9004) -> Option<&'a AgentSpec> {
9005 roster
9006 .get(start + 1..)?
9007 .iter()
9008 .find(|s| !tried.contains(&s.id))
9009}
9010
9011fn pick_successor<'a>(
9032 roster: &'a [AgentSpec],
9033 start: usize,
9034 tried: &BTreeSet<String>,
9035 carried: Option<&BTreeSet<String>>,
9036 others: &BTreeSet<String>,
9037) -> Option<&'a AgentSpec> {
9038 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9039 roster
9040 .get(start + 1..)
9041 .and_then(|tail| {
9042 tail.iter()
9043 .filter(free)
9044 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9045 })
9046 .or_else(|| {
9047 carried
9048 .is_some()
9049 .then(|| roster.iter().find(free))
9050 .flatten()
9051 })
9052 .or_else(|| match carried {
9053 Some(c) => next_for_seat(roster, start, tried, c),
9054 None => next_untried_in_roster(roster, start, tried),
9055 })
9056}
9057
9058fn next_for_seat<'a>(
9068 roster: &'a [AgentSpec],
9069 start: usize,
9070 round_tried: &BTreeSet<String>,
9071 carried_failed: &BTreeSet<String>,
9072) -> Option<&'a AgentSpec> {
9073 roster
9074 .get(start + 1..)?
9075 .iter()
9076 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9077 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9078}
9079
9080fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9086 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9087 return spec;
9088 };
9089 let ok = |id: &str| !h.failed.contains(id);
9090 if let Some(last) = h.last_ok.as_deref()
9091 && ok(last)
9092 && let Some(s) = roster.iter().find(|s| s.id == last)
9093 {
9094 return s.clone();
9095 }
9096 if ok(&spec.id) {
9097 return spec;
9098 }
9099 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9100 roster
9101 .get(from + 1..)
9102 .into_iter()
9103 .flatten()
9104 .chain(roster.iter())
9105 .find(|s| ok(&s.id))
9106 .cloned()
9107 .unwrap_or(spec)
9108}
9109
9110pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9114 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9115}
9116
9117const TIMED_OUT: &str = "timed out";
9120
9121impl FailClass {
9122 fn of(out: &AgentOutcome) -> Option<Self> {
9124 match out {
9125 AgentOutcome::Ok(_) => None,
9126 AgentOutcome::Quota(_) => Some(Self::Quota),
9127 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9128 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9129 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9130 AgentOutcome::ResourceBlocked(_) => None,
9133 }
9134 }
9135
9136 fn stem_word(&self) -> &'static str {
9138 match self {
9139 Self::Quota => "quota",
9140 _ => "handover",
9141 }
9142 }
9143}
9144
9145fn failure_signature(msg: &str) -> String {
9149 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9150 if let Some(code) = line
9153 .strip_prefix("process exited with no usable output (exit code ")
9154 .and_then(|s| s.strip_suffix(')'))
9155 .and_then(|s| s.parse::<i32>().ok())
9156 {
9157 line = format!("exited with some({code}) and no usable output");
9158 } else if matches!(
9159 line.as_str(),
9160 "process exited with no usable output (no exit code reported)"
9161 | "process exited with no usable output (exit code unavailable)"
9162 ) {
9163 line = "exited with none and no usable output".to_owned();
9164 }
9165 let mut out = Vec::new();
9166 for word in line.split_whitespace() {
9167 if word.contains('/') || word.contains('\\') {
9168 out.push("<path>".to_owned());
9169 continue;
9170 }
9171 let mut w = String::new();
9172 let mut in_digits = false;
9173 for c in word.chars() {
9174 if c.is_ascii_digit() {
9175 if !in_digits {
9176 w.push('#');
9177 }
9178 in_digits = true;
9179 } else {
9180 in_digits = false;
9181 w.push(c);
9182 }
9183 }
9184 out.push(w);
9185 }
9186 out.join(" ").chars().take(120).collect()
9187}
9188
9189fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9196 match cur {
9197 FailClass::Quota | FailClass::Timeout => true,
9198 FailClass::Other(_) => prev != Some(cur),
9199 }
9200}
9201
9202fn fail_reason(out: &AgentOutcome) -> String {
9204 match out {
9205 AgentOutcome::Ok(_) => String::new(),
9206 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9207 AgentOutcome::Dropped(o) => format!(
9208 "the CLI dropped the stream ({})",
9209 o.dropped
9210 .as_ref()
9211 .map(|d| d.why.as_str())
9212 .unwrap_or("it ended without delivering its answer")
9213 ),
9214 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9215 e.lines().next().unwrap_or("").chars().take(160).collect()
9216 }
9217 }
9218}
9219
9220pub(crate) fn record_handover(
9223 state: &mut RunState,
9224 node: &str,
9225 seat: &str,
9226 from: &str,
9227 to: &str,
9228 class: &FailClass,
9229 reason: &str,
9230) {
9231 let message = if *class == FailClass::Quota {
9232 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9233 } else {
9234 format!("{seat}: handed over {from} -> {to} ({reason})")
9235 };
9236 state.event(node, message);
9237 state.handovers.push(Handover {
9238 at: Timestamp::now(),
9239 node: node.to_owned(),
9240 seat: seat.to_owned(),
9241 from: from.to_owned(),
9242 to: to.to_owned(),
9243 reason: reason.to_owned(),
9244 });
9245}
9246
9247fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9261 commands.iter().any(|c| c.exit_code.is_none())
9262}
9263
9264fn verified_noop_claim(
9277 usable: bool,
9278 commands: &[agent::CommandEvidence],
9279 text: &str,
9280) -> Option<String> {
9281 (usable && !has_unconfirmed_command(commands))
9282 .then(|| verdict::verified_noop(text))
9283 .flatten()
9284}
9285
9286fn short(commit: &str) -> String {
9287 commit.chars().take(7).collect()
9288}
9289
9290fn make_executable(path: &Path) -> Result<()> {
9291 #[cfg(unix)]
9292 {
9293 use std::os::unix::fs::PermissionsExt as _;
9294 let mut perms = std::fs::metadata(path)?.permissions();
9295 perms.set_mode(0o755);
9296 std::fs::set_permissions(path, perms)?;
9297 }
9298 #[cfg(not(unix))]
9299 {
9300 let _ = path;
9301 }
9302 Ok(())
9303}
9304
9305struct WaveCtx<'a> {
9312 run: &'a str,
9315 node: &'a str,
9317 prompts: &'a Prompts,
9318 cache: Option<&'a Path>,
9320 round: Option<usize>,
9323 carry_seats: bool,
9327 pauses: Option<(&'a Pause, &'a Pause)>,
9330}
9331
9332fn repository_wait(
9334 state: &RunState,
9335 pauses: Option<(&Pause, &Pause)>,
9336) -> crate::repo_lock::WaitPolicy {
9337 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9338 let inherited = crate::repo_lock::WaitPolicy::current();
9339 crate::repo_lock::WaitPolicy::new(
9340 Duration::from_secs(state.config.graph.timeout_resource_wait),
9341 move || {
9342 inherited.cancelled()
9343 || pauses
9344 .as_ref()
9345 .is_some_and(|(a, b)| a.parked() || b.parked())
9346 },
9347 )
9348}
9349
9350fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9351 if error.is::<crate::repo_lock::ResourceBlocked>() {
9352 state.resource_block = Some(crate::run::ResourceBlock {
9353 cause: crate::run::ResourceBlockCause::Other,
9354 node: "repository".into(),
9355 kind: if error.to_string().contains("disk pressure") {
9356 "disk pressure"
9357 } else {
9358 "repository coordination"
9359 }
9360 .into(),
9361 detail: error.to_string(),
9362 at: Timestamp::now(),
9363 });
9364 state.event("resource", error.to_string());
9365 }
9366}
9367
9368async fn coordinate_operation(
9369 state: &mut RunState,
9370 cwd: &Path,
9371 wait: &crate::repo_lock::WaitPolicy,
9372) -> Result<Option<crate::repo_lock::SeatLease>> {
9373 let result =
9374 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9375 .await;
9376 if let Err(e) = &result {
9377 record_repository_block(state, e);
9378 }
9379 result
9380}
9381
9382async fn run_one(
9384 job: SeatJob,
9385 sem: Arc<Semaphore>,
9386 ctx: &WaveCtx<'_>,
9387 state: &mut RunState,
9388 attempt: usize,
9389) -> (SeatState, AgentOutcome) {
9390 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9391 .await
9392 .pop()
9393 .expect("one job in, one result out");
9394 (seat, out)
9395}
9396
9397async fn wave(
9403 jobs: Vec<SeatJob>,
9404 sem: Arc<Semaphore>,
9405 ctx: &WaveCtx<'_>,
9406 state: &mut RunState,
9407 attempt: usize,
9408) -> Vec<(usize, SeatState, AgentOutcome)> {
9409 let WaveCtx {
9410 run,
9411 node,
9412 prompts,
9413 cache,
9414 round,
9415 carry_seats: _,
9416 pauses,
9417 } = *ctx;
9418 for job in &jobs {
9419 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9420 }
9421 if let Err(e) = state.save() {
9422 tracing::warn!("could not persist in-progress seats: {e:#}");
9427 }
9428 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9444 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9448 let cache_guard = if let Some(cache_dir) = cache {
9449 if jobs_had_a_writer {
9450 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9451 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9452 match acquire_cache_lease(
9453 state,
9454 cache_dir,
9455 &owner,
9456 resource_limit,
9457 node,
9458 node,
9459 &cancel,
9460 )
9461 .await
9462 {
9463 Ok(Some(g)) => Some(g),
9464 Ok(None) => {
9467 if let Some(block) = &mut state.resource_block {
9468 block.cause = crate::run::ResourceBlockCause::Cancelled {
9469 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9470 };
9471 }
9472 return block_wave(
9473 jobs,
9474 state,
9475 node,
9476 attempt,
9477 "parked while waiting for the shared build cache",
9478 );
9479 }
9480 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9483 Err(e) => {
9484 let why = format!("{e:#}");
9488 state.event(
9489 "resource",
9490 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9491 );
9492 return block_wave(jobs, state, node, attempt, &why);
9493 }
9494 }
9495 } else {
9496 None
9497 }
9498 } else {
9499 None
9500 };
9501 state.seats_spawning(node, attempt);
9504 if let Err(e) = state.save() {
9505 tracing::warn!("could not persist the start of a wave: {e:#}");
9506 }
9507 let slot_limit = resource_limit
9511 + jobs
9512 .iter()
9513 .map(|j| j.timeout)
9514 .max()
9515 .unwrap_or_default()
9516 .saturating_mul(jobs.len() as u32);
9517 let mut set = tokio::task::JoinSet::new();
9518 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9521 let overlay = prompts.overlay(node);
9522 let repository_wait = repository_wait(state, pauses);
9523 for (i, mut job) in jobs.into_iter().enumerate() {
9524 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9525 if cache.is_some() {
9526 job.prompt.push('\n');
9527 job.prompt
9528 .push_str(&prompt::build_cache_note(node, job.allow_write));
9529 }
9530 let sem = Arc::clone(&sem);
9531 let run = run.to_owned();
9532 let node = node.to_owned();
9533 let attachments = if node == "implement" {
9536 state.attachments.clone()
9537 } else {
9538 Vec::new()
9539 };
9540 let cache = cache
9551 .filter(|_| job.allow_write && cache_guard.is_some())
9552 .map(Path::to_path_buf);
9553 let spawned_tx = spawned_tx.clone();
9554 let repository_wait = repository_wait.clone();
9555 let coordination = state.config.coordination.clone();
9556 let fetched_base = state.fetched_base.clone();
9557 set.spawn(async move {
9558 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9562 Ok(p) => p,
9563 Err(_) => {
9564 return (
9565 i,
9566 job.seat,
9567 AgentOutcome::ResourceBlocked(format!(
9568 "no parallel slot became free within {}s",
9569 slot_limit.as_secs()
9570 )),
9571 false,
9572 );
9573 }
9574 };
9575 spawned_tx.send(job.seat.key.clone()).ok();
9576 let mut seat = job.seat;
9577 let out = repository_wait
9578 .scope(agent::invoke(
9579 &job.spec,
9580 &mut seat,
9581 &Invocation {
9582 coordination: coordination.as_ref(),
9583 fetched_base: fetched_base.as_ref(),
9584 cwd: &job.cwd,
9585 prompt: &job.prompt,
9586 timeout: job.timeout,
9587 allow_write: job.allow_write,
9588 unsandboxed: false,
9589 sessions: job.sessions,
9590 artifacts: &job.artifacts,
9591 stem: &job.stem,
9592 run: &run,
9593 node: &node,
9594 cache_dir: cache.as_deref(),
9595 attachments: &attachments,
9596 writable: &[],
9597 },
9598 ))
9599 .await;
9600 let interrupted = out
9601 .as_ref()
9602 .err()
9603 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9604 let out = match out {
9605 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9606 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9607 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9615 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9616 Ok(o) => AgentOutcome::Failed(format!(
9617 "process exited with no usable output ({})",
9618 crate::notices::exit_status(o.exit_code)
9619 )),
9620 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9621 AgentOutcome::ResourceBlocked(e.to_string())
9622 }
9623 Err(e) => AgentOutcome::Failed(e.to_string()),
9624 };
9625 (i, seat, out, interrupted)
9626 });
9627 }
9628 drop(spawned_tx);
9629 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9630 loop {
9631 let joined = tokio::select! {
9632 Some(key) = spawned_rx.recv() => {
9633 state.seat_spawned(&key);
9635 if let Err(e) = state.save() {
9636 tracing::warn!("could not persist a seat's start: {e:#}");
9637 }
9638 continue;
9639 }
9640 joined = set.join_next() => joined,
9641 };
9642 let Some(joined) = joined else {
9643 break;
9644 };
9645 let (i, seat, out, interrupted) = match joined {
9646 Ok(v) => v,
9647 Err(e) => {
9651 tracing::error!("agent task panicked: {e}");
9652 continue;
9653 }
9654 };
9655 state.seat_finished(&seat.key);
9656 if interrupted
9657 && node == "implement"
9658 && let Some(label) = state
9659 .candidates
9660 .iter()
9661 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9662 .map(|candidate| candidate.label)
9663 {
9664 state.candidate_preservation.insert(
9665 label,
9666 "interrupted implementation awaiting persistence".into(),
9667 );
9668 state.interrupted_implementations.insert(label);
9669 }
9670 if let AgentOutcome::ResourceBlocked(why) = &out
9671 && state.resource_block.is_none()
9672 {
9673 state.resource_block = Some(crate::run::ResourceBlock {
9674 cause: crate::run::ResourceBlockCause::Other,
9675 node: node.to_owned(),
9676 kind: if why.contains("disk pressure") {
9677 "disk pressure"
9678 } else if why.contains("repository coordination") {
9679 "repository coordination"
9680 } else {
9681 "parallel slot"
9682 }
9683 .to_owned(),
9684 detail: why.clone(),
9685 at: Timestamp::now(),
9686 });
9687 let activity = if interrupted {
9688 "agent execution interrupted"
9689 } else {
9690 "no agent was started"
9691 };
9692 state.event("resource", format!("{node}: {activity} - {why}"));
9693 }
9694 record_jobs(state, node, round, &seat.key, &out);
9695 if let Err(e) = state.save() {
9696 tracing::warn!("could not persist a seat's completion: {e:#}");
9697 }
9698 if collected.len() <= i {
9699 collected.resize_with(i + 1, || None);
9700 }
9701 collected[i] = Some((i, seat, out));
9702 }
9703 if state
9709 .active
9710 .values()
9711 .any(|a| a.node == node && a.attempt == attempt)
9712 {
9713 state
9714 .active
9715 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9716 if let Err(e) = state.save() {
9717 tracing::warn!("could not persist the end of a wave: {e:#}");
9718 }
9719 }
9720 if let Some(cache_dir) = cache
9727 && jobs_had_a_writer
9728 {
9729 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9730 }
9731 if let Some(guard) = cache_guard {
9732 guard.release();
9733 }
9734 collected.into_iter().flatten().collect()
9735}
9736
9737fn record_jobs(
9748 state: &mut RunState,
9749 node: &str,
9750 round: Option<usize>,
9751 seat: &str,
9752 out: &AgentOutcome,
9753) {
9754 let commands: &[agent::CommandEvidence] = match out {
9755 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9756 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9757 };
9758 let checked_at = Timestamp::now();
9759 for c in commands {
9760 state.jobs.push(JobRecord {
9761 node: node.to_owned(),
9762 round,
9763 seat: seat.to_owned(),
9764 id: c.id.clone(),
9765 description: c.description.clone(),
9766 checked_at,
9767 status: match c.exit_code {
9768 Some(0) => JobStatus::Completed,
9769 Some(_) => JobStatus::Failed,
9770 None => JobStatus::Unknown,
9771 },
9772 exit_code: c.exit_code,
9773 result_summary: c.result_summary.clone(),
9774 source: c.source.clone(),
9775 });
9776 }
9777}
9778
9779fn round_is_clean(
9800 blocking: usize,
9801 e2e_ok: bool,
9802 answered: usize,
9803 expected: usize,
9804 quota_missing: usize,
9805 policy: IncompleteReviewPolicy,
9806) -> bool {
9807 if blocking != 0 || !e2e_ok {
9808 return false;
9809 }
9810 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9811 return true;
9812 }
9813 answered > 0 && expected - answered <= quota_missing
9814}
9815
9816pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9840 if max_rounds == 0
9841 || reviews.last().is_some_and(|r| {
9842 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9843 })
9844 {
9845 return Some(RunStatus::Gating);
9846 }
9847 let last = reviews.last()?;
9848 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9849 if reviews.len() < max_rounds && !stagnant {
9850 return None;
9851 }
9852 if last.incomplete() && last.blocking == 0 {
9853 return Some(RunStatus::Blocked);
9854 }
9855 if last.e2e_status() == E2eStatus::ResourceBlocked
9856 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9857 {
9858 return None;
9859 }
9860 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9861 RunStatus::Gating
9862 } else {
9863 RunStatus::Blocked
9864 })
9865}
9866
9867pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9882 if nudged {
9883 (full / 4).max(Duration::from_secs(120)).min(full)
9884 } else {
9885 full
9886 }
9887}
9888
9889#[allow(clippy::too_many_arguments)]
9918async fn ask_json_wave<T>(
9919 jobs: Vec<SeatJob>,
9920 sem: Arc<Semaphore>,
9921 retries: usize,
9922 roster: &[AgentSpec],
9923 ctx: &WaveCtx<'_>,
9924 losses: &mut Vec<QuotaLoss>,
9925 state: &mut RunState,
9926 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9927) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9928where
9929 T: serde::de::DeserializeOwned + Send + 'static,
9930{
9931 ask_wave_with(
9932 jobs,
9933 sem,
9934 retries,
9935 roster,
9936 ctx,
9937 losses,
9938 state,
9939 &|text: &str| {
9940 let v = verdict::extract_json::<T>(text)?;
9941 validate(&v)?;
9942 Ok(v)
9943 },
9944 )
9945 .await
9946}
9947
9948#[allow(clippy::too_many_arguments)]
9957async fn ask_wave_with<T>(
9958 jobs: Vec<SeatJob>,
9959 sem: Arc<Semaphore>,
9960 retries: usize,
9961 roster: &[AgentSpec],
9962 ctx: &WaveCtx<'_>,
9963 losses: &mut Vec<QuotaLoss>,
9964 state: &mut RunState,
9965 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9966) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9967where
9968 T: Send + 'static,
9969{
9970 let n = jobs.len();
9971 let originals: Vec<SeatJob> = jobs;
9972 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9973 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9974 let mut nudges: Vec<usize> = vec![0; n];
9981 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9985 let mut tried: Vec<BTreeSet<String>> = specs
9986 .iter()
9987 .map(|s| BTreeSet::from([s.id.clone()]))
9988 .collect();
9989 let starts: Vec<usize> = specs
9990 .iter()
9991 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9992 .collect();
9993 let carry = ctx.carry_seats && !roster.is_empty();
9994 let carried: Vec<BTreeSet<String>> = originals
9997 .iter()
9998 .map(|j| {
9999 state
10000 .seat_history
10001 .get(&j.seat.key)
10002 .filter(|_| carry)
10003 .map(|h| h.failed.clone())
10004 .unwrap_or_default()
10005 })
10006 .collect();
10007 let mut prev: Vec<Option<FailClass>> = originals
10008 .iter()
10009 .map(|j| {
10010 state
10011 .seat_history
10012 .get(&j.seat.key)
10013 .filter(|_| carry)
10014 .and_then(|h| h.last_fail.clone())
10015 })
10016 .collect();
10017 let next_agent =
10018 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10019 let others: BTreeSet<String> = specs
10020 .iter()
10021 .enumerate()
10022 .filter(|(j, _)| *j != i)
10023 .map(|(_, s)| s.id.clone())
10024 .collect();
10025 pick_successor(
10026 roster,
10027 starts[i],
10028 tried,
10029 carry.then(|| &carried[i]),
10030 &others,
10031 )
10032 .cloned()
10033 };
10034 let mut fresh: Vec<Option<String>> = vec![None; n];
10035 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10036 let mut pending: Vec<usize> = (0..n).collect();
10037
10038 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10043 for round in 0..max_rounds {
10044 if pending.is_empty() {
10045 break;
10046 }
10047 let mut batch = Vec::with_capacity(pending.len());
10048 let mut renudged: Vec<&str> = Vec::new();
10049 for &i in &pending {
10050 let src = &originals[i];
10051 let full: &str = match &src.handover {
10056 Some(h) if specs[i].id != src.spec.id => h,
10057 _ => &src.prompt,
10058 };
10059 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10062 (
10063 full.to_owned(),
10064 src.timeout,
10065 format!("{}-{word}-{}", src.stem, specs[i].id),
10066 )
10067 } else if nudges[i] == 0 {
10068 (full.to_owned(), src.timeout, src.stem.clone())
10069 } else {
10070 renudged.push(src.seat.key.as_str());
10071 let why = done[i]
10072 .as_ref()
10073 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10074 .unwrap_or_else(|| "no parsable answer".to_owned());
10075 let nudge = prompt::nudge(&why);
10076 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10077 let prompt = if nudged {
10078 nudge
10079 } else {
10080 format!("{full}\n\n---\n\n{nudge}")
10081 };
10082 (
10083 prompt,
10084 retry_budget(src.timeout, nudged),
10085 format!("{}-retry{}", src.stem, nudges[i]),
10086 )
10087 };
10088 batch.push(SeatJob {
10089 spec: specs[i].clone(),
10090 seat: seats[i].clone(),
10091 cwd: src.cwd.clone(),
10092 prompt,
10093 timeout,
10094 allow_write: src.allow_write,
10095 sessions: src.sessions,
10096 artifacts: src.artifacts.clone(),
10097 stem,
10098 handover: None,
10099 });
10100 }
10101
10102 if !renudged.is_empty() {
10103 state.event(
10104 ctx.node,
10105 format!("retry {round}: re-asking {}", renudged.join(", ")),
10106 );
10107 }
10108 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10109 let mut still = Vec::new();
10110 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10111 seats[i] = seat;
10112 if let AgentOutcome::ResourceBlocked(why) = &out {
10115 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10116 continue;
10117 }
10118 let class = FailClass::of(&out);
10119 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10122 && nudges[i] < retries
10123 && !roster.is_empty();
10124 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10125 let next = should_hand_over(prev[i].as_ref(), &cur)
10126 .then(|| next_agent(i, &tried[i], &specs))
10127 .flatten();
10128 if carry {
10129 let h = state
10130 .seat_history
10131 .entry(originals[i].seat.key.clone())
10132 .or_default();
10133 h.failed.insert(specs[i].id.clone());
10134 h.last_fail = Some(cur.clone());
10135 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10136 h.last_ok = None;
10137 }
10138 if let Err(e) = state.save() {
10141 tracing::warn!("could not persist a seat's failure history: {e:#}");
10142 }
10143 }
10144 if let Some(next) = next {
10145 record_handover(
10146 state,
10147 ctx.node,
10148 &originals[i].seat.key,
10149 &specs[i].id,
10150 &next.id,
10151 &cur,
10152 &fail_reason(&out),
10153 );
10154 tried[i].insert(next.id.clone());
10155 prev[i] = Some(cur.clone());
10156 seats[i] =
10157 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10158 specs[i] = next;
10159 fresh[i] = Some(cur.stem_word().to_owned());
10160 nudges[i] = 0;
10161 done[i] = Some(Err(anyhow::anyhow!(
10162 "handed over after: {}",
10163 fail_reason(&out)
10164 )));
10165 still.push(i);
10166 continue;
10167 }
10168 }
10169 if carry
10170 && !nudge_first
10171 && let Some(cur) = class.clone()
10172 {
10173 let h = state
10175 .seat_history
10176 .entry(originals[i].seat.key.clone())
10177 .or_default();
10178 h.failed.insert(specs[i].id.clone());
10179 h.last_fail = Some(cur);
10180 }
10181 let parsed = match out {
10182 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10183 AgentOutcome::Quota(o) => {
10184 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10185 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10186 }
10187 AgentOutcome::Dropped(o) => {
10191 let why = o
10192 .dropped
10193 .as_ref()
10194 .map(|d| d.why.as_str())
10195 .unwrap_or("the CLI ended the stream without delivering its answer");
10196 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10197 }
10198 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10199 Err(anyhow::anyhow!(e))
10200 }
10201 };
10202 let quota = class == Some(FailClass::Quota);
10203 let failed = parsed.is_err();
10204 done[i] = Some(parsed);
10205 if carry && !failed {
10206 let h = state
10207 .seat_history
10208 .entry(originals[i].seat.key.clone())
10209 .or_default();
10210 h.failed.remove(&specs[i].id);
10211 h.last_ok = Some(specs[i].id.clone());
10212 h.last_fail = None;
10213 }
10214 let agent_failure = class.is_some()
10222 && !nudge_first
10223 && !roster.is_empty()
10224 && next_agent(i, &tried[i], &specs).is_some();
10225 if failed && !quota && !agent_failure && nudges[i] < retries {
10226 nudges[i] += 1;
10227 still.push(i);
10228 }
10229 }
10230 pending = still;
10231 }
10232
10233 for (i, q) in last_quota.into_iter().enumerate() {
10236 if let Some(reset) = q {
10237 losses.push(QuotaLoss {
10238 seat: originals[i].seat.key.clone(),
10239 node: ctx.node.to_owned(),
10240 at: Timestamp::now(),
10241 reset,
10242 });
10243 }
10244 }
10245
10246 seats
10247 .into_iter()
10248 .zip(done)
10249 .zip(nudges)
10250 .map(|((seat, res), attempts)| {
10251 (
10252 seat,
10253 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10254 attempts,
10255 )
10256 })
10257 .collect()
10258}
10259
10260fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10262 results
10263 .iter()
10264 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10265}
10266
10267fn block_wave(
10271 jobs: Vec<SeatJob>,
10272 state: &mut RunState,
10273 node: &str,
10274 attempt: usize,
10275 why: &str,
10276) -> Vec<(usize, SeatState, AgentOutcome)> {
10277 state.resource_wait = None;
10278 state
10279 .active
10280 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10281 if let Err(e) = state.save() {
10282 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10283 }
10284 jobs.into_iter()
10285 .enumerate()
10286 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10287 .collect()
10288}
10289
10290async fn acquire_cache_lease(
10303 state: &mut RunState,
10304 cache_dir: &Path,
10305 owner: &crate::cache::Owner,
10306 budget: Duration,
10307 node: &str,
10308 context: &str,
10309 cancel: &(dyn Fn() -> bool + Sync),
10310) -> Result<Option<crate::cache::Guard>> {
10311 let home = crate::run::home();
10312 let started = Instant::now();
10313 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10314 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10315 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10316 Err(e) => {
10317 state.event(
10318 "verify",
10319 format!("{context}: could not check the shared build cache: {e:#}"),
10320 );
10321 if let Err(e2) = state.save() {
10322 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10323 }
10324 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10325 }
10326 };
10327 state.event(
10328 "verify",
10329 format!(
10330 "{context}: waiting for the shared build cache at {} ({})",
10331 cache_dir.display(),
10332 busy.describe()
10333 ),
10334 );
10335 state.resource_wait = Some(crate::run::ResourceWait {
10338 node: context.to_owned(),
10339 kind: "shared build cache".to_owned(),
10340 holder: busy.describe(),
10341 since: Timestamp::now(),
10342 limit_secs: budget.as_secs(),
10343 });
10344 if let Err(e) = state.save() {
10345 tracing::warn!("could not persist a cache wait: {e:#}");
10346 }
10347 let remaining = budget.saturating_sub(started.elapsed());
10348 let waited = crate::cache::wait_for_unless(
10349 &home,
10350 cache_dir,
10351 owner,
10352 remaining,
10353 Duration::from_secs(5),
10354 cancel,
10355 )
10356 .await;
10357 state.resource_wait = None;
10358 let cause = match &waited {
10359 Ok(Some(_)) => None,
10360 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10361 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10362 Some(crate::run::ResourceBlockCause::Timeout {
10363 waited_secs: started.elapsed().as_secs(),
10364 limit_secs: budget.as_secs(),
10365 })
10366 }
10367 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10368 };
10369 if let Some(cause) = cause {
10370 let detail = match &waited {
10371 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10372 Err(error) => format!("{error:#}"),
10373 };
10374 state.resource_block = Some(crate::run::ResourceBlock {
10375 cause,
10376 node: node.to_owned(),
10377 kind: "shared build cache".into(),
10378 detail,
10379 at: Timestamp::now(),
10380 });
10381 state.event(
10382 "resource",
10383 state.resource_block.as_ref().unwrap().description(),
10384 );
10385 }
10386 if let Err(error) = state.save() {
10387 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10388 }
10389 waited
10390}
10391
10392#[derive(Debug)]
10397struct CacheCheckError(String);
10398
10399impl std::fmt::Display for CacheCheckError {
10400 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10401 f.write_str(&self.0)
10402 }
10403}
10404
10405impl std::error::Error for CacheCheckError {}
10406
10407fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10409 state
10410 .resource_block
10411 .get_or_insert_with(|| crate::run::ResourceBlock {
10412 cause: crate::run::ResourceBlockCause::Other,
10413 node: node.to_owned(),
10414 kind: "shared build cache".into(),
10415 detail: format!("{error:#}"),
10416 at: Timestamp::now(),
10417 });
10418 if let Err(error) = state.save() {
10419 tracing::warn!("could not persist cache resource block: {error:#}");
10420 }
10421}
10422
10423#[allow(clippy::too_many_arguments)]
10444async fn with_cache_lease<'s, F, Fut>(
10445 state: &'s mut RunState,
10446 cache_dir: Option<&Path>,
10447 node: &str,
10448 seat: &str,
10449 worktree: &Path,
10450 head: &str,
10451 budget: Duration,
10452 context: &str,
10453 wait: &crate::repo_lock::WaitPolicy,
10454 body: F,
10455) -> Result<(Vec<CommandOutcome>, bool)>
10456where
10457 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10458 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10459{
10460 let started = Instant::now();
10463 let home = crate::run::home();
10464 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10465 let guard = if let Some(cache_dir) = cache_dir {
10466 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10467 match acquire_cache_lease(
10468 state,
10469 cache_dir,
10470 &owner,
10471 resource_limit,
10472 node,
10473 context,
10474 &|| wait.cancelled(),
10475 )
10476 .await
10477 .and_then(|g| {
10478 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10479 }) {
10480 Ok(g) => Some(g),
10481 Err(e) => {
10482 record_cache_error(state, node, &e);
10483 return Ok((
10484 vec![CommandOutcome {
10485 command: "(waiting for the shared build cache)".into(),
10486 code: None,
10487 output_tail: e.to_string(),
10488 duration_ms: started.elapsed().as_millis() as u64,
10489 resource_blocked: true,
10490 }],
10491 false,
10492 ));
10493 }
10494 }
10495 } else {
10496 None
10497 };
10498 let _coordination = coordinate_operation(state, worktree, wait).await?;
10499 if let Some(cache_dir) = cache_dir {
10500 let identity = crate::cache::Identity::new(worktree, head);
10501 if let Err(e) = crate::cache::ensure_fresh(
10502 &home,
10503 cache_dir,
10504 &identity,
10505 &state.config.verify.cargo_manifests(),
10506 ) {
10507 record_cache_error(state, node, &e);
10508 state.event(
10509 "verify",
10510 format!(
10511 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10512 worktree.display(),
10513 short(head)
10514 ),
10515 );
10516 if let Some(guard) = guard {
10517 guard.release();
10518 }
10519 return Ok((
10520 vec![CommandOutcome {
10521 command: "(confirming the shared build cache is fresh)".into(),
10522 code: None,
10523 output_tail: e.to_string(),
10524 duration_ms: started.elapsed().as_millis() as u64,
10525 resource_blocked: true,
10526 }],
10527 false,
10528 ));
10529 }
10530 }
10531 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10532 if !timed_out_pids.is_empty() {
10533 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10534 }
10535 drop(_coordination);
10538 if let Some(guard) = guard {
10539 guard.release();
10540 }
10541 Ok((outcomes, retried))
10542}
10543
10544async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10556 wait_for_pids_with(
10557 pids,
10558 crate::proc::pid_alive,
10559 LEASE_RELEASE_POLL,
10560 LEASE_RELEASE_MAX_WAIT,
10561 )
10562 .await;
10563}
10564
10565async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10571 pids: &[u32],
10572 alive: F,
10573 poll: Duration,
10574 max_wait: Duration,
10575) {
10576 let deadline = Instant::now() + max_wait;
10577 loop {
10578 if pids.iter().all(|&pid| !alive(pid)) {
10579 return;
10580 }
10581 if Instant::now() >= deadline {
10582 return;
10583 }
10584 tokio::time::sleep(poll).await;
10585 }
10586}
10587
10588fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10595 outcomes.iter().any(|o| o.resource_blocked)
10596}
10597
10598pub(crate) enum GateFix {
10600 Retry,
10602 Stop,
10605 Defer,
10608}
10609
10610fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10618 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10619 red.peek().is_some()
10620 && red.all(|o| {
10621 !o.resource_blocked
10622 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10623 && !o.output_tail.trim().is_empty()
10624 })
10625}
10626
10627fn e2e_outcome_label(o: &CommandOutcome) -> String {
10631 if o.ok() {
10632 return "pass".to_owned();
10633 }
10634 let reason = if o.build_failed() {
10635 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10636 } else {
10637 format!("FAIL ({:?})", o.code)
10638 };
10639 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10640}
10641
10642async fn run_e2e_with_retry(
10650 state: &mut RunState,
10651 shell: &[String],
10652 commands: &[String],
10653 worktree: &Path,
10654 timeout: Duration,
10655 context: &str,
10656) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10657 let (mut e2e, mut timed_out_pids) = run_commands(
10658 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10659 )
10660 .await;
10661 for o in &e2e {
10662 state.event(
10663 "verify",
10664 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10665 );
10666 }
10667 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10671 if verify_retried {
10672 state.event(
10673 "verify",
10674 format!(
10675 "{context}: verify could not build/link, not a test result — retrying once \
10676 before concluding"
10677 ),
10678 );
10679 let retried = run_commands(
10680 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10681 )
10682 .await;
10683 e2e = retried.0;
10684 timed_out_pids.extend(retried.1);
10687 for o in &e2e {
10688 state.event(
10689 "verify",
10690 format!(
10691 "{context}: retry `{}` -> {}",
10692 o.command,
10693 e2e_outcome_label(o)
10694 ),
10695 );
10696 }
10697 }
10698 (e2e, verify_retried, timed_out_pids)
10699}
10700
10701#[allow(clippy::too_many_arguments)]
10717async fn run_commands(
10718 state: &mut RunState,
10719 node: &str,
10720 task: &str,
10721 attempt: usize,
10722 shell: &[String],
10723 commands: &[String],
10724 cwd: &Path,
10725 timeout: Duration,
10726) -> (Vec<CommandOutcome>, Vec<u32>) {
10727 if commands.is_empty() {
10728 return (Vec::new(), Vec::new());
10733 }
10734 let mut out = Vec::new();
10735 let mut timed_out_pids = Vec::new();
10736 let total = commands.len();
10737 for (idx, command) in commands.iter().enumerate() {
10738 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10739 if let Err(e) = state.save() {
10740 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10741 }
10742 let started = Instant::now();
10743 let mut cmd = tokio::process::Command::new(&shell[0]);
10744 cmd.quiet();
10745 for key in crate::worker::INVOCATION_ENV {
10749 cmd.env_remove(key);
10750 }
10751 cmd.args(&shell[1..])
10752 .arg(command)
10753 .current_dir(cwd)
10754 .stdin(std::process::Stdio::null())
10755 .stdout(std::process::Stdio::piped())
10756 .stderr(std::process::Stdio::piped())
10757 .kill_on_drop(true);
10758 let mut outputs = Vec::new();
10759 let protected = (|| -> Result<()> {
10760 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10761 outputs.push(guard);
10762 }
10763 if let Some(cache) = state.config.cache_dir() {
10764 outputs.push(crate::retention::OutputLease::cache(
10765 &crate::run::home(),
10766 &cache,
10767 false,
10768 )?);
10769 }
10770 Ok(())
10771 })();
10772 if let Err(e) = protected {
10773 out.push(CommandOutcome {
10774 command: command.clone(),
10775 code: None,
10776 output_tail: format!("output ownership unavailable: {e:#}"),
10777 duration_ms: 0,
10778 resource_blocked: true,
10779 });
10780 continue;
10781 }
10782 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10783 let (code, body) = match spawned {
10784 Ok(mut running) => {
10785 let pid = running.child().id();
10786 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10787 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10788 let mut paths = vec![cwd.to_owned()];
10789 if let Some(cache) = state.config.cache_dir() {
10790 paths.push(cache);
10791 }
10792 let mut pressure = None;
10793 let status = tokio::select! {
10794 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10795 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10796 pressure = Some(reason); None
10797 }
10798 };
10799 let (code, failure) = match status {
10800 Some(Ok(Ok(status))) => (
10801 status.code(),
10802 status
10803 .code()
10804 .is_none()
10805 .then(|| format!("command terminated: {status}")),
10806 ),
10807 Some(Ok(Err(error))) => {
10808 (None, Some(format!("failed waiting for command: {error}")))
10809 }
10810 Some(Err(_)) => {
10811 if !running.coordinated()
10815 && let Some(pid) = pid
10816 {
10817 timed_out_pids.push(pid);
10818 }
10819 (
10820 None,
10821 Some(format!("timed out after {}s", timeout.as_secs())),
10822 )
10823 }
10824 None => (None, None),
10825 };
10826 running.finish().await;
10827 if let Some(reason) = pressure {
10828 state.resource_block = Some(crate::run::ResourceBlock {
10829 cause: crate::run::ResourceBlockCause::Other,
10830 node: node.to_owned(),
10831 kind: "disk pressure".to_owned(),
10832 detail: reason.clone(),
10833 at: Timestamp::now(),
10834 });
10835 out.push(CommandOutcome {
10836 command: command.clone(),
10837 code: None,
10838 output_tail: reason,
10839 duration_ms: started.elapsed().as_millis() as u64,
10840 resource_blocked: true,
10841 });
10842 break;
10843 }
10844 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10845 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10846 if let Some(failure) = failure {
10847 body.push('\n');
10848 body.push_str(&failure);
10849 }
10850 (code, body)
10851 }
10852 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10853 };
10854 out.push(CommandOutcome {
10855 command: command.clone(),
10856 code,
10857 output_tail: tail(&body, OUTPUT_TAIL),
10858 duration_ms: started.elapsed().as_millis() as u64,
10859 resource_blocked: false,
10860 });
10861 }
10862 state.task_finished(task);
10863 if let Err(e) = state.save() {
10864 tracing::warn!("could not persist the end of {task}: {e:#}");
10865 }
10866 (out, timed_out_pids)
10867}
10868
10869fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10881 let repo = repo.display();
10882 match style {
10883 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10884 MergeStyle::Squash => {
10885 let subject = message
10888 .lines()
10889 .next()
10890 .unwrap_or(branch)
10891 .replace(['\\', '"', '$', '`'], "");
10892 format!(
10893 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10894 )
10895 }
10896 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10897 }
10898}
10899
10900const PR_TITLE_MAX: usize = 240;
10911
10912fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10923 let line = queue::first_line(text)?;
10924 let chars: Vec<char> = line.chars().collect();
10925 let (kept, mark) = if chars.len() <= max {
10926 (line.to_owned(), "")
10927 } else if let Some(end) = sentence_end(&chars, max) {
10928 (chars[..end].iter().collect::<String>(), "")
10929 } else {
10930 let room = max.saturating_sub(3);
10931 let cut = if chars[room].is_whitespace() {
10934 room
10935 } else {
10936 chars[..room]
10937 .iter()
10938 .rposition(|c| c.is_whitespace())
10939 .unwrap_or(room)
10940 };
10941 let head: String = chars[..cut].iter().collect();
10942 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10943 (head.to_owned(), "...")
10944 };
10945 if kept.is_empty() {
10946 return None;
10947 }
10948 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10949 return None;
10950 }
10951 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10952 return None;
10953 }
10954 Some(format!("{kept}{mark}"))
10955}
10956
10957fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10961 const MIN: usize = 20;
10962 for i in MIN..max.min(chars.len()) {
10963 if !matches!(chars[i], '.' | '!' | '?') {
10964 continue;
10965 }
10966 let Some(&next) = chars.get(i + 1) else {
10967 continue;
10968 };
10969 if !next.is_whitespace() {
10970 continue;
10971 }
10972 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10973 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10974 continue;
10975 }
10976 let word: String = chars[..i]
10977 .iter()
10978 .rev()
10979 .take_while(|c| !c.is_whitespace())
10980 .collect::<Vec<_>>()
10981 .into_iter()
10982 .rev()
10983 .collect();
10984 let word = word.to_ascii_lowercase();
10985 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10986 continue;
10987 }
10988 let end = chars[..i]
10989 .iter()
10990 .rposition(|c| !c.is_whitespace())
10991 .map_or(i, |p| p + 1);
10992 return Some(end);
10993 }
10994 None
10995}
10996
10997struct PrMessage {
11002 title: String,
11003 body: String,
11004}
11005
11006impl PrMessage {
11007 fn commit_message(&self) -> String {
11011 format!("{}\n\n{}", self.title, self.body)
11012 }
11013}
11014
11015fn title_marker(line: &str) -> Option<&str> {
11017 let line = line.trim();
11018 let head = line.get(..6)?;
11019 head.eq_ignore_ascii_case("title:")
11020 .then(|| line[6..].trim())
11021}
11022
11023fn summary_title(summary: &str) -> Option<String> {
11028 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11029 let raw = title_marker(first)?;
11030 if raw.is_empty() {
11031 return None;
11032 }
11033 let title = queue::title_from(raw, PR_TITLE_MAX);
11034 let lower = title.to_ascii_lowercase();
11035 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11036 return None;
11037 }
11038 Some(title)
11039}
11040
11041const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11043
11044pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11046
11047pub(crate) fn is_review_run(state: &RunState) -> bool {
11051 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11052}
11053
11054fn review_title(state: &RunState) -> Option<String> {
11059 english_subject(
11060 state.reviewed_commits.as_ref()?.first()?,
11061 state.config.graph.github_english(),
11062 )
11063}
11064
11065fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11068 let raw = raw.trim();
11069 if raw.is_empty() {
11070 return None;
11071 }
11072 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11073 return None;
11074 }
11075 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11076 return None;
11077 }
11078 let title = queue::title_from(raw, PR_TITLE_MAX);
11079 let lower = title.to_ascii_lowercase();
11080 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11081 return None;
11082 }
11083 Some(title)
11084}
11085
11086#[derive(Debug, Clone, PartialEq, Eq)]
11089struct BranchFacts {
11090 commits: Vec<(String, String)>,
11092 stat: String,
11094}
11095
11096const STAT_FILE_LINES: usize = 25;
11098const COMMITS_MAX_CHARS: usize = 20_000;
11100
11101async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11104 let commits = git::commit_log(repo, base, branch).await.ok()?;
11105 if commits.is_empty() {
11106 return None;
11107 }
11108 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11109 let lines: Vec<&str> = stat.lines().collect();
11110 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11111 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11112 let more = format!(" ... {omitted} more file(s)");
11113 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11114 kept.push(&more);
11115 kept.push(lines[lines.len() - 1]);
11116 kept.join("\n")
11117 } else {
11118 lines.join("\n")
11119 };
11120 Some(BranchFacts { commits, stat })
11121}
11122
11123fn markdown_safe(text: &str) -> String {
11126 text.replace("</details>", "</details>")
11127 .replace("\x60\x60\x60", "~~~")
11128}
11129
11130fn neutral_title(state: &RunState, winner: char) -> String {
11131 format!(
11132 "chore: land candidate {} of run {}",
11133 winner.to_ascii_uppercase(),
11134 state.id
11135 )
11136}
11137
11138pub(crate) async fn repair_landing_gate(
11141 state: &mut RunState,
11142 winner: &Candidate,
11143 outcomes: &[CommandOutcome],
11144) -> Result<GateFix> {
11145 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11146 return Ok(GateFix::Stop);
11147 }
11148 let mut runner = Runner {
11149 roles: state.config.resolve_roles()?,
11150 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11151 state: state.clone(),
11152 pause: Pause::new(),
11153 interrupt: Pause::new(),
11154 };
11155 let result = runner.gate_fix_round(winner, outcomes).await;
11156 *state = runner.state;
11157 result
11158}
11159
11160pub(crate) async fn check_gate_repair_e2e(
11163 state: &mut RunState,
11164 winner: &Candidate,
11165 wait: &crate::repo_lock::WaitPolicy,
11166) -> Result<GateFix> {
11167 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11168 return Ok(GateFix::Retry);
11169 }
11170 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11171 if state
11172 .gate_repair_e2e
11173 .as_ref()
11174 .is_some_and(|r| r.head == head && r.passed)
11175 {
11176 return Ok(GateFix::Retry);
11177 }
11178 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11179 head: head.clone(),
11180 passed: false,
11181 outcomes: Vec::new(),
11182 });
11183 state.save()?;
11184 let commands = state.config.verify.e2e.clone();
11185 let outcomes = if commands.is_empty() {
11186 Vec::new()
11187 } else {
11188 let shell = state.config.shell();
11189 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11190 let cache_dir = state.config.cache_dir();
11191 let (outcomes, _) = with_cache_lease(
11192 state,
11193 cache_dir.as_deref(),
11194 "e2e",
11195 "e2e",
11196 &winner.worktree,
11197 &head,
11198 timeout,
11199 "gate repair e2e",
11200 wait,
11201 |state, budget| async move {
11202 run_e2e_with_retry(
11203 state,
11204 &shell,
11205 &commands,
11206 &winner.worktree,
11207 budget,
11208 "gate repair e2e",
11209 )
11210 .await
11211 },
11212 )
11213 .await?;
11214 outcomes
11215 };
11216 let deferred = verify_inconclusive(&outcomes);
11217 let passed = outcomes.iter().all(CommandOutcome::ok)
11218 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11219 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11220 head,
11221 passed,
11222 outcomes,
11223 });
11224 state.save()?;
11225 if deferred {
11226 Ok(GateFix::Defer)
11227 } else if !passed {
11228 state.event(
11229 "gate",
11230 "verify.e2e failed after gate repair; nothing published",
11231 );
11232 Ok(GateFix::Stop)
11233 } else {
11234 Ok(GateFix::Retry)
11235 }
11236}
11237
11238pub(crate) async fn run_landing_gate(
11240 state: &mut RunState,
11241 winner: &Candidate,
11242 wait: &crate::repo_lock::WaitPolicy,
11243) -> Result<Vec<CommandOutcome>> {
11244 let shell = state.config.shell();
11245 let gate_commands = state.config.verify.gate.clone();
11246 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11247 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11248 Vec::new()
11249 } else {
11250 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11251 let cache_dir = state.config.cache_dir();
11252 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11253 let (outcomes, _) = with_cache_lease(
11254 state,
11255 cache_dir.as_deref(),
11256 "gate",
11257 "gate",
11258 &winner.worktree,
11259 &head,
11260 timeout,
11261 "final gate",
11262 wait,
11263 |state, budget| {
11264 let shell = shell.clone();
11265 let gate_commands = gate_commands.clone();
11266 let winner = winner.clone();
11267 async move {
11268 Runner::run_pre_gate(state, &winner).await;
11269 let (outcomes, timed_out_pids) = run_commands(
11270 state,
11271 "gate",
11272 "gate",
11273 0,
11274 &shell,
11275 &gate_commands,
11276 &winner.worktree,
11277 budget,
11278 )
11279 .await;
11280 (outcomes, false, timed_out_pids)
11281 }
11282 },
11283 )
11284 .await?;
11285 outcomes
11286 };
11287 if outcomes.is_empty() {
11288 state.event("gate", "no gate commands configured; not tested");
11293 }
11294 for o in &outcomes {
11295 state.event(
11296 "gate",
11297 format!(
11298 "`{}` -> {}",
11299 o.command,
11300 if o.ok() {
11301 "pass".to_owned()
11302 } else {
11303 format!(
11304 "FAIL ({:?})\n{}",
11305 o.code,
11306 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11307 )
11308 }
11309 ),
11310 );
11311 }
11312 Ok(outcomes)
11313}
11314
11315pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11321 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11322 ""
11323 } else {
11324 pr_title
11325 }
11326}
11327
11328pub fn landing_subject_source(state: &RunState) -> String {
11332 if is_review_run(state) {
11333 let winner = state.candidates.first().map_or('A', |c| c.label);
11334 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11335 }
11336 state.instruction.clone()
11337}
11338
11339fn summary_without_title(summary: &str) -> String {
11342 let mut lines = summary.trim().lines().peekable();
11343 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11344 lines.next();
11345 }
11346 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11347}
11348
11349#[cfg(test)]
11361fn pr_message(state: &RunState, winner: char) -> PrMessage {
11362 pr_message_with(state, winner, None)
11363}
11364
11365#[cfg(test)]
11368fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11369 let raw = pr_message_raw(
11370 state,
11371 winner,
11372 facts,
11373 crate::pr_outcome::FinalHead::Unreported,
11374 ChangeNote::NotAsked,
11375 );
11376 let id = crate::scrub::Identity::current();
11377 PrMessage {
11378 title: crate::scrub::scrub(&raw.title, &id),
11379 body: crate::scrub::scrub(&raw.body, &id),
11380 }
11381}
11382
11383#[derive(Debug, Clone, Copy)]
11385enum ChangeNote<'a> {
11386 NotAsked,
11388 Summary(&'a crate::run::ChangeSummary),
11390 Unavailable(&'a str),
11392}
11393
11394fn pr_message_raw(
11395 state: &RunState,
11396 winner: char,
11397 facts: Option<&BranchFacts>,
11398 head: crate::pr_outcome::FinalHead<'_>,
11399 change: ChangeNote<'_>,
11400) -> PrMessage {
11401 if !matches!(change, ChangeNote::NotAsked) {
11402 let (title, mut body) = match change {
11403 ChangeNote::Summary(c) => (
11404 c.title.clone(),
11405 format!("## Summary\n\n{}\n\n", c.body.trim()),
11406 ),
11407 ChangeNote::Unavailable(why) => (
11408 neutral_title(state, winner),
11409 format!(
11410 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11411 ),
11412 ),
11413 ChangeNote::NotAsked => unreachable!(),
11414 };
11415 body.push_str(&public_pr_evidence(state, head));
11416 body.push_str(&format!(
11417 "\n---\nmagi:run/{} magi:candidate-{}\n",
11418 state.id,
11419 winner.to_ascii_lowercase()
11420 ));
11421 return PrMessage { title, body };
11422 }
11423 pr_preview(state, winner, facts, head)
11424}
11425
11426fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11429 let mut body = crate::pr_outcome::review_rounds(state);
11430 body.push_str(&crate::pr_outcome::final_verification(state, head));
11431 let open = state.last_round_findings();
11432 if !open.is_empty() {
11433 body.push_str("## Open review findings\n\n");
11434 for f in open {
11435 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11436 }
11437 body.push('\n');
11438 }
11439 if let Some(c) = state.change_summary.as_ref()
11440 && head == crate::pr_outcome::FinalHead::At(&c.head)
11441 {
11442 body.push_str(&format!(
11443 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11444 c.base, c.head
11445 ));
11446 }
11447 body
11448}
11449
11450fn pr_preview(
11452 state: &RunState,
11453 winner: char,
11454 facts: Option<&BranchFacts>,
11455 head: crate::pr_outcome::FinalHead<'_>,
11456) -> PrMessage {
11457 let summary = state
11458 .candidates
11459 .iter()
11460 .find(|c| c.label == winner)
11461 .map(|c| c.summary.as_str())
11462 .unwrap_or_default();
11463 let review = is_review_run(state);
11466 let title = if review {
11467 facts
11468 .and_then(|f| {
11469 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11470 })
11471 .or_else(|| review_title(state))
11472 .or_else(|| {
11473 state
11474 .candidates
11475 .iter()
11476 .find(|c| c.label == winner)
11477 .filter(|c| !c.branch.starts_with("magi/"))
11478 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11479 })
11480 .unwrap_or_else(|| neutral_title(state, winner))
11481 } else {
11482 summary_title(summary).unwrap_or_else(|| {
11483 english_title(
11484 &state.instruction,
11485 PR_TITLE_MAX,
11486 state.config.graph.github_english(),
11487 )
11488 .unwrap_or_else(|| neutral_title(state, winner))
11489 })
11490 };
11491
11492 let mut body = String::new();
11493 let what = summary_without_title(summary);
11494 if !what.is_empty() {
11495 body.push_str("## Summary\n\n");
11496 body.push_str(&what);
11497 body.push_str("\n\n");
11498 }
11499
11500 body.push_str(&crate::pr_outcome::review_rounds(state));
11502 body.push_str(&crate::pr_outcome::final_verification(state, head));
11503 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11504
11505 let open = state.open_findings();
11506 if !open.is_empty() {
11507 body.push_str("## Open review findings\n\n");
11508 for f in &open {
11509 body.push_str(&format!(
11510 "- `{}` [{:?}] {}\n",
11511 f.id, f.severity, "Context retained in local review records."
11512 ));
11513 }
11514 body.push('\n');
11515 }
11516
11517 if let Some(fix) = fix
11518 && !fix.rejected.is_empty()
11519 {
11520 body.push_str("## Declined by the fixer\n\n");
11521 for r in &fix.rejected {
11522 body.push_str(&format!(
11523 "- `{}`: {}\n",
11524 r.id, "Reason retained in local review records."
11525 ));
11526 }
11527 body.push('\n');
11528 }
11529
11530 if review {
11531 body.push_str("## Commits under review\n\n");
11533 if let Some(facts) = facts {
11534 let mut left = COMMITS_MAX_CHARS;
11535 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11536 let mut entry = format!("- {}\n", markdown_safe(subject));
11537 for l in markdown_safe(text).lines() {
11538 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11539 }
11540 if left == 0 {
11541 body.push_str(&format!(
11542 "- ... {} more commit(s)\n",
11543 facts.commits.len() - i
11544 ));
11545 break;
11546 }
11547 if entry.len() > left {
11548 let mut end = left;
11551 while !entry.is_char_boundary(end) {
11552 end -= 1;
11553 }
11554 entry.truncate(end);
11555 entry.push_str("\n ... (truncated)\n");
11556 left = 0;
11557 } else {
11558 left -= entry.len();
11559 }
11560 body.push_str(&entry);
11561 }
11562 if !facts.stat.trim().is_empty() {
11563 body.push_str(&format!(
11564 "\n## Diff stat\n\n```\n{}\n```\n",
11565 markdown_safe(facts.stat.trim())
11566 ));
11567 }
11568 } else {
11569 match &state.reviewed_commits {
11570 Some(subjects) => {
11571 for s in subjects {
11572 body.push_str(&format!("- {}\n", s.trim()));
11573 }
11574 }
11575 None => {
11576 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11579 body.push_str(rest.trim());
11580 body.push('\n');
11581 }
11582 }
11583 }
11584 } else {
11585 let task = state.instruction.trim();
11586 let task = if task.is_empty() {
11587 "(empty task)"
11588 } else {
11589 task
11590 };
11591 body.push_str(&format!(
11592 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11593 task.replace("</details>", "</details>")
11594 ));
11595 }
11596
11597 body.push_str(&format!(
11598 "\n---\nmagi:run/{} magi:candidate-{}\n",
11599 state.id,
11600 winner.to_ascii_lowercase()
11601 ));
11602
11603 PrMessage { title, body }
11604}
11605
11606fn seeded_instruction(state: &RunState) -> String {
11609 let mut inputs = Vec::new();
11610 if let Some(start) = &state.sources.start_from {
11611 inputs.push(format!(
11612 "Start from `{}` at `{}` (including ancestry).",
11613 start.token, start.sha
11614 ));
11615 }
11616 for pick in &state.sources.pick_commits {
11617 inputs.push(format!(
11618 "Pick `{}` at `{}` (single commit).",
11619 pick.token, pick.sha
11620 ));
11621 }
11622 if inputs.is_empty() {
11623 state.instruction.clone()
11624 } else {
11625 format!(
11626 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11627 state.instruction,
11628 inputs.join("\n")
11629 )
11630 }
11631}
11632
11633async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11637 let base = &state.base_branch;
11638 let mut against = base.clone();
11639 if mode == MergeMode::Pr {
11640 let remote = &state.config.merge.remote;
11644 let tracking = format!("{remote}/{base}");
11645 let fetched = git::fetch(repo, remote, base).await;
11646 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11647 against = tracking;
11648 } else {
11649 return false;
11650 }
11651 }
11652 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11653}
11654
11655fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11658 let mut detail = format!(
11659 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11660 nothing to open a pull request for"
11661 );
11662 detail.push_str(
11663 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11664 );
11665 detail
11666}
11667
11668#[derive(Debug, PartialEq, Eq)]
11671enum PrPlan {
11672 Create,
11673 Adopt { url: String, title: String },
11674 Stop(String),
11675}
11676
11677fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11680 match found {
11681 Ok(land::OpenPr::None) => PrPlan::Create,
11682 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11683 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11684 "several open pull requests exist for this branch, not picking one: {}",
11685 urls.join(" ")
11686 )),
11687 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11688 }
11689}
11690
11691pub(crate) async fn gh_pr_create(
11693 cwd: &Path,
11694 base: &str,
11695 head: &str,
11696 title: &str,
11697 body: &str,
11698) -> Result<String> {
11699 let out = tokio::process::Command::new("gh")
11700 .args([
11701 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11702 ])
11703 .current_dir(cwd)
11704 .quiet()
11705 .stdin(std::process::Stdio::null())
11706 .output()
11707 .await
11708 .context("spawn gh")?;
11709 if out.status.success() {
11710 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11711 } else {
11712 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11713 }
11714}
11715
11716async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11718 match std::fs::symlink_metadata(path) {
11719 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11720 git::worktree_remove_clean(repo, path).await?;
11723 return Ok(());
11724 }
11725 Err(e) => return Err(e.into()),
11726 Ok(_) => {}
11727 }
11728 if !git::is_clean(path).await? {
11729 bail!(
11730 "worktree has uncommitted work; refusing removal: {}",
11731 path.display()
11732 );
11733 }
11734 if !git::worktree_remove_clean(repo, path).await? {
11735 bail!("worktree removal refused: {}", path.display());
11736 }
11737 Ok(())
11738}
11739
11740pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11749 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11750 bail!("refusing fold: {reason}");
11751 }
11752 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11753 bail!("candidate preservation incomplete; refusing cleanup");
11754 }
11755 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11759 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11760 let _driver = crate::driver::maintenance(home, &state.id)?;
11761 let mut output_guards = Vec::new();
11762 for entry in std::fs::read_dir(state.worktree_root())
11763 .into_iter()
11764 .flatten()
11765 .flatten()
11766 {
11767 if entry.path().join(".git").is_file() {
11768 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11769 }
11770 }
11771 let repo = state.repo.clone();
11772 let wait = repository_wait(state, None);
11775 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11776 Ok(lease) => lease,
11777 Err(e) => {
11778 state.save_under(home)?;
11779 return Err(e);
11780 }
11781 };
11782 let _source_outputs = if state.status == RunStatus::Merged
11785 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11786 {
11787 verify_merged_fold_sources(state, drop_winner, home).await?
11788 } else {
11789 Vec::new()
11790 };
11791 let root = state.worktree_root();
11792 let winner = state.tally.as_ref().map(|t| t.winner);
11793 let mut removed = Vec::new();
11794
11795 for i in 0..state.candidates.len() {
11796 let c = state.candidates[i].clone();
11797 let is_winner = Some(c.label) == winner;
11798 if is_winner && !drop_winner {
11799 continue;
11800 }
11801 let existed = c.worktree.exists();
11802 remove_preserved_worktree(&repo, &c.worktree).await?;
11805 if existed {
11806 removed.push(c.worktree.to_string_lossy().into_owned());
11807 }
11808 let handed_over = state.released_branches.contains(&c.branch);
11811 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11812 if !git::branch_delete(&repo, &c.branch).await? {
11813 bail!("candidate branch removal refused");
11814 }
11815 removed.push(c.branch.clone());
11816 }
11817 state.candidates[i].folded = true;
11818 }
11819
11820 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11821 let path = name.path();
11822 let keep = !drop_winner
11823 && winner.is_some_and(|w| {
11824 path.file_name()
11825 .is_some_and(|n| n == format!("cand-{w}").as_str())
11826 });
11827 if keep {
11828 continue;
11829 }
11830 remove_preserved_worktree(&repo, &path).await?;
11831 removed.push(path.to_string_lossy().into_owned());
11832 }
11833
11834 remove_if_empty(&root);
11843
11844 if state.enabled_worktree_config && drop_winner {
11845 git::release_worktree_config(&repo).await.ok();
11849 state.enabled_worktree_config = false;
11850 }
11851 state.save_under(home)?;
11852 Ok(removed)
11853}
11854
11855async fn verify_merged_fold_sources(
11856 state: &RunState,
11857 drop_winner: bool,
11858 home: &Path,
11859) -> Result<Vec<std::fs::File>> {
11860 let winner = state.tally.as_ref().map(|t| t.winner);
11861 let mut heads = BTreeSet::new();
11862 for candidate in &state.candidates {
11863 if Some(candidate.label) == winner && !drop_winner {
11864 continue;
11865 }
11866 anyhow::ensure!(
11867 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11868 "candidate recovery source is outside the inventoried run root: {}",
11869 candidate.worktree.display()
11870 );
11871 match std::fs::symlink_metadata(&candidate.worktree) {
11872 Ok(_) => {}
11873 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11874 Err(error) => return Err(error.into()),
11875 }
11876 if !state.released_branches.contains(&candidate.branch)
11877 && git::branch_exists(&state.repo, &candidate.branch).await?
11878 {
11879 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11880 }
11881 }
11882 let mut cargo_guards = Vec::new();
11883 let entries = match std::fs::read_dir(state.worktree_root()) {
11884 Ok(entries) => Some(entries),
11885 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11886 Err(error) => return Err(error.into()),
11887 };
11888 for entry in entries.into_iter().flatten() {
11889 let path = entry?.path();
11890 let metadata = std::fs::symlink_metadata(&path)?;
11891 anyhow::ensure!(
11892 metadata.is_dir() && !metadata.file_type().is_symlink(),
11893 "non-directory or symlink recovery source retained: {}",
11894 path.display()
11895 );
11896 if !drop_winner
11897 && winner.is_some_and(|w| {
11898 path.file_name()
11899 .is_some_and(|name| name == format!("cand-{w}").as_str())
11900 })
11901 {
11902 continue;
11903 }
11904 anyhow::ensure!(
11905 path.join(".git").is_file(),
11906 "unregistered recovery source retained: {}",
11907 path.display()
11908 );
11909 anyhow::ensure!(
11910 git::git(
11911 &path,
11912 &[
11913 "status",
11914 "--porcelain",
11915 "--untracked-files=all",
11916 "--ignored",
11917 "--",
11918 ".",
11919 ":(exclude)target"
11920 ]
11921 )
11922 .await?
11923 .is_empty(),
11924 "source changes or ignored recovery files retained: {}",
11925 path.display()
11926 );
11927 match std::fs::symlink_metadata(path.join("target")) {
11928 Ok(_) => {
11929 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11930 }
11931 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11932 Err(error) => return Err(error.into()),
11933 }
11934 for checkpoint in [
11935 "rebase-merge",
11936 "rebase-apply",
11937 "sequencer",
11938 "CHERRY_PICK_HEAD",
11939 "MERGE_HEAD",
11940 "REBASE_HEAD",
11941 "REVERT_HEAD",
11942 "BISECT_LOG",
11943 "AUTO_MERGE",
11944 "MERGE_MSG",
11945 ] {
11946 let location = git::git(
11947 &path,
11948 &[
11949 "rev-parse",
11950 "--path-format=absolute",
11951 "--git-path",
11952 checkpoint,
11953 ],
11954 )
11955 .await?;
11956 anyhow::ensure!(
11957 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11958 if e.kind() == std::io::ErrorKind::NotFound),
11959 "Git recovery checkpoint retained: {checkpoint}"
11960 );
11961 }
11962 heads.insert(git::rev_parse(&path, "HEAD").await?);
11963 }
11964 crate::fold_evidence::preserve(state, home, &heads).await?;
11965
11966 Ok(cargo_guards)
11967}
11968
11969fn remove_if_empty(dir: &Path) {
11980 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11981 std::fs::remove_dir(dir).ok();
11982 }
11983}
11984
11985pub fn worst_open(state: &RunState) -> Option<Severity> {
11987 state
11988 .reviews
11989 .last()?
11990 .reviews
11991 .iter()
11992 .flat_map(|r| r.findings.iter())
11993 .map(|f| f.severity)
11994 .max()
11995}
11996
11997async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12001 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12002 git::acquire_worktree_config(repo).await?;
12003 state.enabled_worktree_config = true;
12004 }
12005 Ok(())
12006}
12007
12008#[cfg(test)]
12009mod tests {
12010 #[tokio::test]
12011 async fn launcher_base_fetch_requires_remote_success() {
12012 let tmp = tempfile::tempdir().unwrap();
12013 let repo = tmp.path().join("repo");
12014 let remote = tmp.path().join("remote");
12015 std::fs::create_dir(&repo).unwrap();
12016 std::fs::create_dir(&remote).unwrap();
12017 git::git(&remote, &["init", "--bare", "-b", "main"])
12018 .await
12019 .unwrap();
12020 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12021 git::git(
12022 &repo,
12023 &[
12024 "-c",
12025 "user.name=Fixture",
12026 "-c",
12027 "user.email=fixture@example.invalid",
12028 "commit",
12029 "--allow-empty",
12030 "-m",
12031 "fixture",
12032 ],
12033 )
12034 .await
12035 .unwrap();
12036 git::git(
12037 &repo,
12038 &["remote", "add", "origin", remote.to_str().unwrap()],
12039 )
12040 .await
12041 .unwrap();
12042 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12043 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12044 assert_eq!(
12045 resolve_base(&repo, "main", "origin", None)
12046 .await
12047 .unwrap()
12048 .commit,
12049 head
12050 );
12051 #[cfg(unix)]
12052 {
12053 let policy: crate::config::Coordination = toml::from_str(
12054 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12055 ).unwrap();
12056 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12057 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12058 let held = crate::repo_lock::SeatLease::operation_with(
12059 &repo,
12060 Some(&policy),
12061 &crate::repo_lock::WaitPolicy::default(),
12062 )
12063 .await
12064 .unwrap();
12065 assert!(held.is_some());
12066 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12067 tokio::pin!(fetch);
12068 assert!(
12069 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12070 .await
12071 .is_err()
12072 );
12073 assert!(
12074 !repo.join(".git/FETCH_HEAD").exists(),
12075 "no fetch before admission"
12076 );
12077 drop(held);
12078 let fetched = fetch.await.unwrap();
12079 assert_eq!(fetched.commit, head);
12080 assert_eq!(fetched.remote, "origin");
12081 assert_eq!(fetched.branch, "main");
12082 }
12083 git::git(
12085 &repo,
12086 &[
12087 "remote",
12088 "set-url",
12089 "origin",
12090 tmp.path().join("missing").to_str().unwrap(),
12091 ],
12092 )
12093 .await
12094 .unwrap();
12095 let error = resolve_base(&repo, "main", "origin", None)
12096 .await
12097 .unwrap_err()
12098 .to_string();
12099 assert!(error.contains("cannot read origin/main"));
12100 assert!(error.contains("refusing to branch off"));
12101 }
12102
12103 #[test]
12104 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12105 let leaked = vec!["chore(deps): update a crate".to_owned()];
12106 let own = "fix(daemon): apply a chosen action";
12107 assert!(should_retitle("", own, &leaked));
12108 assert!(should_retitle(
12109 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12110 own,
12111 &leaked
12112 ));
12113 assert!(should_retitle(
12114 "chore: land candidate A of run 1",
12115 own,
12116 &leaked
12117 ));
12118 assert!(should_retitle(
12119 "magi: candidate A (uncommitted work)",
12120 own,
12121 &leaked
12122 ));
12123 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12124 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12125 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12126 }
12127
12128 #[test]
12129 fn pr_merge_plan_creates_adopts_or_stops() {
12130 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12131 assert_eq!(
12132 pr_merge_plan(Ok(land::OpenPr::One {
12133 url: "u".into(),
12134 title: "t".into()
12135 })),
12136 PrPlan::Adopt {
12137 url: "u".into(),
12138 title: "t".into()
12139 }
12140 );
12141 let PrPlan::Stop(many) =
12142 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12143 else {
12144 panic!("many must stop");
12145 };
12146 assert!(many.contains('a') && many.contains('b'));
12147 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12148 panic!("a failed lookup must stop");
12149 };
12150 assert!(err.contains("bad token"));
12151 }
12152
12153 use super::*;
12154 use crate::run::GateStatus;
12155 use std::collections::BTreeMap;
12156 use std::time::Duration;
12157
12158 fn conductor() -> AgentSpec {
12159 AgentSpec {
12160 id: "conductor".to_owned(),
12161 kind: crate::config::AgentKind::Command,
12162 model: None,
12163 command: vec!["true".to_owned()],
12164 extra_args: Vec::new(),
12165 env: BTreeMap::new(),
12166 prompt_delivery: None,
12167 max_turn_secs: None,
12168 }
12169 }
12170
12171 fn spec(id: &str) -> AgentSpec {
12172 AgentSpec {
12173 id: id.to_owned(),
12174 kind: crate::config::AgentKind::Command,
12175 model: None,
12176 command: vec!["true".to_owned()],
12177 extra_args: Vec::new(),
12178 env: BTreeMap::new(),
12179 prompt_delivery: None,
12180 max_turn_secs: None,
12181 }
12182 }
12183
12184 fn ids(xs: &[&str]) -> BTreeSet<String> {
12185 xs.iter().map(|s| (*s).to_owned()).collect()
12186 }
12187
12188 #[test]
12189 fn pick_successor_skips_an_agent_another_seat_holds() {
12190 let roster = [spec("a"), spec("b"), spec("c")];
12192 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12193 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12194 }
12195
12196 #[test]
12197 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12198 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12200 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12201 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12202 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12203 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12204 }
12205
12206 #[test]
12207 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12208 let roster = [spec("a"), spec("b")];
12209 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12210 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12211 let carried = ids(&["b"]);
12212 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12213 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12214 }
12215
12216 #[test]
12217 fn pick_successor_returns_none_without_any_untried_successor() {
12218 let roster = [spec("a"), spec("b")];
12219 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12220 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12221 let carried = ids(&["a"]);
12222 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12223 }
12224
12225 #[test]
12226 fn pick_successor_rescue_avoids_another_seats_occupant() {
12227 let roster = [spec("a"), spec("b"), spec("c")];
12229 let carried = ids(&["b", "c"]);
12230 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12231 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12232 }
12233
12234 #[test]
12235 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12236 let roster = [spec("a"), spec("b"), spec("c")];
12237 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12238 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12239 }
12240
12241 #[test]
12242 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12243 let roster = [spec("a"), spec("b"), spec("c")];
12244 let failed = ids(&["a", "b", "c"]);
12245 let mut tried = ids(&["b"]);
12247 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12248 assert_eq!(first.id, "a");
12249 tried.insert(first.id.clone());
12250 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12251 assert_eq!(second.id, "c");
12252 tried.insert(second.id.clone());
12253 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12254 }
12255
12256 #[test]
12257 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12258 let roster = [spec("a"), spec("b")];
12259 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12260 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12261 }
12262
12263 #[test]
12264 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12265 let roster = [spec("a"), spec("b"), spec("c")];
12266 let h = SeatHistory {
12267 failed: ids(&["a"]),
12268 last_ok: Some("b".to_owned()),
12269 last_fail: None,
12270 };
12271 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12272 let gone = SeatHistory {
12274 last_ok: Some("zzz".to_owned()),
12275 ..h.clone()
12276 };
12277 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12278 let failed = SeatHistory {
12279 failed: ids(&["a", "b"]),
12280 last_ok: Some("b".to_owned()),
12281 last_fail: None,
12282 };
12283 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12284 }
12285
12286 #[test]
12287 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12288 let first = SeatState::new("review-1", "alpha", 7);
12289 let next = handover_seat("review-1", "gamma", 7);
12290 assert_ne!(first.claude_session, next.claude_session);
12291 }
12292
12293 #[test]
12294 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12295 let mut state = state_with_summary("x", "y");
12296 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12297 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12298 assert_ne!(a.claude_session, b.claude_session);
12299 }
12300
12301 #[test]
12302 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12303 let roster = [spec("a"), spec("b")];
12304 let h = SeatHistory {
12305 failed: ids(&["a", "b"]),
12306 ..SeatHistory::default()
12307 };
12308 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12309 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12310 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12311 }
12312
12313 #[test]
12320 fn failure_signature_ignores_numbers_and_paths() {
12321 assert_eq!(
12322 failure_signature("exited with Some(2) and no usable output"),
12323 failure_signature("exited with Some(137) and no usable output")
12324 );
12325 assert_eq!(
12326 failure_signature("process exited with no usable output (exit code 2)"),
12327 failure_signature("process exited with no usable output (exit code 137)")
12328 );
12329 for (old, new) in [
12330 (
12331 "exited with Some(2) and no usable output",
12332 "process exited with no usable output (exit code 2)",
12333 ),
12334 (
12335 "exited with None and no usable output",
12336 "process exited with no usable output (exit code unavailable)",
12337 ),
12338 ] {
12339 let previous = FailClass::Other(failure_signature(old));
12340 let current = FailClass::Other(failure_signature(new));
12341 assert_eq!(previous, current, "resumption retains the failure class");
12342 assert!(!should_hand_over(Some(&previous), ¤t));
12343 }
12344 assert_eq!(
12345 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12346 failure_signature("cannot open /var/x.txt: denied")
12347 );
12348 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12349 }
12350
12351 #[test]
12352 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12353 let other = FailClass::Other("x".into());
12354 assert!(should_hand_over(None, &FailClass::Quota));
12355 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12356 assert!(should_hand_over(
12357 Some(&FailClass::Timeout),
12358 &FailClass::Timeout
12359 ));
12360 assert!(should_hand_over(None, &other));
12361 assert!(!should_hand_over(Some(&other), &other));
12362 assert!(should_hand_over(
12363 Some(&other),
12364 &FailClass::Other("y".into())
12365 ));
12366 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12368 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12369 }
12370
12371 #[test]
12372 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12373 let a = SeatState::new("judge-1", "alpha", 7);
12374 let b = handover_seat("judge-1", "beta", 7);
12375 assert_ne!(a.claude_session, b.claude_session);
12376 assert_eq!(b.turns, 0);
12377 }
12378
12379 #[test]
12380 fn a_timeout_is_classified_apart_from_other_failures() {
12381 assert_eq!(
12382 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12383 Some(FailClass::Timeout)
12384 );
12385 assert!(matches!(
12386 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12387 Some(FailClass::Other(_))
12388 ));
12389 }
12390
12391 #[test]
12392 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12393 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12394 let tried = BTreeSet::from(["beta".to_owned()]);
12395 let next = next_untried_in_roster(&roster, 1, &tried);
12398 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12399 }
12400
12401 #[test]
12402 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12403 let roster = vec![spec("alpha"), spec("beta")];
12404 let tried = BTreeSet::from(["beta".to_owned()]);
12405 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12409 }
12410
12411 #[test]
12412 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12413 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12414 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12415 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12419 }
12420
12421 #[test]
12422 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12423 let roster = vec![spec("a"), spec("a"), spec("b")];
12424 let tried = BTreeSet::from(["a".to_owned()]);
12425 let next = next_untried_in_roster(&roster, 0, &tried);
12426 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12427 }
12428
12429 #[test]
12430 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12431 let roster = vec![spec("a"), spec("b")];
12432 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12433 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12434 }
12435
12436 #[test]
12437 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12438 let dir = tempfile::tempdir().unwrap();
12439 let bay = dir.path().join("ffff");
12440
12441 remove_if_empty(&bay);
12443 assert!(!bay.exists());
12444
12445 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12448 remove_if_empty(&bay);
12449 assert!(bay.exists(), "non-empty directory must survive");
12450
12451 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12453 remove_if_empty(&bay);
12454 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12455 }
12456
12457 #[test]
12466 fn a_full_panel_that_found_nothing_is_clean() {
12467 assert!(round_is_clean(
12468 0,
12469 true,
12470 2,
12471 2,
12472 0,
12473 IncompleteReviewPolicy::Block
12474 ));
12475 }
12476
12477 #[test]
12478 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12479 assert!(!round_is_clean(
12480 0,
12481 true,
12482 1,
12483 2,
12484 0,
12485 IncompleteReviewPolicy::Block
12486 ));
12487 }
12488
12489 #[test]
12490 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12491 assert!(!round_is_clean(
12492 1,
12493 true,
12494 1,
12495 2,
12496 0,
12497 IncompleteReviewPolicy::Warn
12498 ));
12499 }
12500
12501 #[test]
12502 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12503 assert!(round_is_clean(
12504 0,
12505 true,
12506 1,
12507 2,
12508 0,
12509 IncompleteReviewPolicy::Warn
12510 ));
12511 }
12512
12513 #[test]
12514 fn a_full_panel_with_an_open_finding_is_not_clean() {
12515 assert!(!round_is_clean(
12516 1,
12517 true,
12518 2,
12519 2,
12520 0,
12521 IncompleteReviewPolicy::Block
12522 ));
12523 }
12524
12525 #[test]
12526 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12527 assert!(!round_is_clean(
12528 0,
12529 false,
12530 2,
12531 2,
12532 0,
12533 IncompleteReviewPolicy::Block
12534 ));
12535 }
12536
12537 #[test]
12544 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12545 assert!(round_is_clean(
12548 0,
12549 true,
12550 1,
12551 2,
12552 1,
12553 IncompleteReviewPolicy::Block
12554 ));
12555 }
12556
12557 #[test]
12558 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12559 assert!(!round_is_clean(
12562 0,
12563 true,
12564 1,
12565 2,
12566 0,
12567 IncompleteReviewPolicy::Block
12568 ));
12569 }
12570
12571 #[test]
12572 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12573 assert!(!round_is_clean(
12574 1,
12575 true,
12576 1,
12577 2,
12578 1,
12579 IncompleteReviewPolicy::Block
12580 ));
12581 assert!(!round_is_clean(
12582 0,
12583 false,
12584 1,
12585 2,
12586 1,
12587 IncompleteReviewPolicy::Block
12588 ));
12589 }
12590
12591 #[test]
12592 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12593 assert!(!round_is_clean(
12597 0,
12598 true,
12599 0,
12600 2,
12601 2,
12602 IncompleteReviewPolicy::Block
12603 ));
12604 }
12605
12606 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12607 CommandOutcome {
12608 command: "test".to_owned(),
12609 code,
12610 output_tail: String::new(),
12611 duration_ms: 0,
12612 resource_blocked,
12613 }
12614 }
12615
12616 #[test]
12617 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12618 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12619 assert!(
12620 !verify_inconclusive(&[outcome(Some(1), false)]),
12621 "an ordinary failure is still evidence about the patch"
12622 );
12623 assert!(verify_inconclusive(&[outcome(None, true)]));
12624 assert!(
12625 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12626 "one inconclusive outcome taints the whole batch"
12627 );
12628 assert!(!verify_inconclusive(&[]));
12629 }
12630
12631 #[tokio::test]
12632 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12633 let calls = std::sync::atomic::AtomicUsize::new(0);
12637 let started = Instant::now();
12638 wait_for_pids_with(
12639 &[123],
12640 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12641 Duration::from_millis(5),
12642 Duration::from_secs(5),
12643 )
12644 .await;
12645 assert!(
12646 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12647 "must keep checking rather than deciding on the first answer"
12648 );
12649 assert!(
12650 started.elapsed() < Duration::from_secs(1),
12651 "must return the moment it is confirmed dead, not wait out the ceiling"
12652 );
12653 }
12654
12655 #[tokio::test]
12656 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12657 let started = Instant::now();
12658 wait_for_pids_with(
12659 &[123],
12660 |_| true, Duration::from_millis(5),
12662 Duration::from_millis(30),
12663 )
12664 .await;
12665 let elapsed = started.elapsed();
12666 assert!(
12667 elapsed >= Duration::from_millis(30),
12668 "must not give up before its own ceiling: {elapsed:?}"
12669 );
12670 assert!(
12671 elapsed < Duration::from_secs(1),
12672 "must not wait past its own ceiling either: {elapsed:?}"
12673 );
12674 }
12675
12676 #[tokio::test]
12677 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12678 let started = Instant::now();
12679 wait_for_pids_with(
12680 &[],
12681 |_| true,
12682 Duration::from_secs(5),
12683 Duration::from_secs(5),
12684 )
12685 .await;
12686 assert!(
12687 started.elapsed() < Duration::from_millis(200),
12688 "an empty pid list has nothing to confirm"
12689 );
12690 }
12691
12692 fn review_round(
12698 clean: bool,
12699 blocking: usize,
12700 answered: usize,
12701 expected: usize,
12702 progressed: bool,
12703 e2e_ok: bool,
12704 ) -> ReviewRound {
12705 ReviewRound {
12706 round: 1,
12707 head: "h".to_owned(),
12708 verified_head: None,
12709 verified_at: None,
12710 reviews: Vec::new(),
12711 e2e: vec![CommandOutcome {
12712 command: "test".to_owned(),
12713 code: Some(if e2e_ok { 0 } else { 1 }),
12714 output_tail: String::new(),
12715 duration_ms: 0,
12716 resource_blocked: false,
12717 }],
12718 verify_retried: false,
12719 e2e_deferred: false,
12720 e2e_defer_reason: None,
12721 fix: None,
12722 blocking,
12723 answered,
12724 expected,
12725 clean,
12726 progressed,
12727 vote_split: false,
12728 reconsideration: Vec::new(),
12729 verdict: None,
12730 }
12731 }
12732
12733 #[test]
12734 fn review_conclusion_is_none_when_nothing_has_run() {
12735 assert_eq!(review_conclusion(&[], 3), None);
12736 }
12737
12738 #[test]
12739 fn review_conclusion_is_none_while_rounds_remain() {
12740 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12741 assert_eq!(review_conclusion(&rounds, 3), None);
12742 }
12743
12744 #[test]
12745 fn review_conclusion_is_gating_once_a_round_is_clean() {
12746 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12747 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12748 }
12749
12750 #[test]
12751 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12752 let rounds = vec![
12753 review_round(false, 1, 2, 2, true, true),
12754 review_round(false, 1, 2, 2, true, true),
12755 ];
12756 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12757 }
12758
12759 #[test]
12760 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12761 let rounds = vec![
12762 review_round(false, 1, 2, 2, true, true),
12763 review_round(false, 1, 2, 2, true, false),
12764 ];
12765 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12766 }
12767
12768 #[test]
12769 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12770 let mut blocked = review_round(false, 1, 2, 2, true, false);
12777 blocked.e2e[0].resource_blocked = true;
12778 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12779 assert_eq!(review_conclusion(&rounds, 2), None);
12780 }
12781
12782 #[test]
12783 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12784 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12786 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12787 }
12788
12789 #[test]
12790 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12791 let rounds = vec![
12792 review_round(false, 1, 2, 2, false, true),
12793 review_round(false, 1, 2, 2, false, true),
12794 ];
12795 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12796 }
12797
12798 fn secs(n: u64) -> Duration {
12799 Duration::from_secs(n)
12800 }
12801
12802 #[tokio::test]
12805 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12806 ask_test_home();
12807 let tmp = tempfile::tempdir().unwrap();
12808 let repo = tmp.path().join("repo");
12809 std::fs::create_dir_all(&repo).unwrap();
12810 init_repo(&repo);
12811 let g = |args: &[&str]| {
12812 let out = std::process::Command::new("git")
12813 .args(args)
12814 .current_dir(&repo)
12815 .quiet()
12816 .output()
12817 .unwrap();
12818 assert!(
12819 out.status.success(),
12820 "{}",
12821 String::from_utf8_lossy(&out.stderr)
12822 );
12823 };
12824 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12825 g(&["checkout", "-b", "historical"]);
12826 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12827 g(&["add", "."]);
12828 g(&["commit", "-m", "old alternative"]);
12829 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12830 g(&["checkout", "-b", "magi/task-selected", &base]);
12831 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12832 g(&["add", "."]);
12833 g(&["commit", "-m", "measurement infrastructure"]);
12834 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12835 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12836 g(&["add", "."]);
12837 g(&["commit", "-m", "dependent implementation"]);
12838 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12839 g(&["checkout", "main"]);
12840 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12841 g(&["add", "."]);
12842 g(&["commit", "-m", "unrelated base change"]);
12843 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12844 let origin = tmp.path().join("origin.git");
12845 g(&[
12846 "clone",
12847 "--bare",
12848 repo.to_str().unwrap(),
12849 origin.to_str().unwrap(),
12850 ]);
12851 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12852 g(&["fetch", "origin"]);
12853 let selection = crate::pr_recovery::Selection {
12854 url: "https://example.test/project/pull/1".into(),
12855 branch: "magi/task-selected".into(),
12856 head: head.clone(),
12857 source_head: None,
12858 source_run: None,
12859 source_proof: None,
12860 disposition: Some(crate::pr_recovery::Disposition::Review),
12861 instruction: "preserve measured behavior".into(),
12862 approved_at: jiff::Timestamp::now(),
12863 };
12864 let mut template = runner_at(RunStatus::Reviewing);
12865 template.state.repo = repo.clone();
12866 template.state.base_commit = current.clone();
12867 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12868 template.state.origin = Some(crate::run::Origin::queue("selected"));
12869 template.state.pr_head = Some(selection.branch.clone());
12870 template.state.pr_recovery = Some(selection.clone());
12871 let mut runner =
12872 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12873 .await
12874 .unwrap();
12875 let wt = runner.state.winner().unwrap().worktree.clone();
12876 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12877 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12878 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12879 assert!(runner.state.seeds.is_empty());
12880 assert_eq!(runner.state.instruction, "preserve measured behavior");
12881 runner.sync_to_base().await.unwrap();
12882 assert_eq!(
12883 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12884 "prerequisite"
12885 );
12886 assert_eq!(
12887 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12888 "measured implementation"
12889 );
12890 assert_eq!(
12891 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12892 "current main change"
12893 );
12894 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12895 assert_eq!(
12896 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12897 head
12898 );
12899
12900 let mut partial = runner_at(RunStatus::Reviewing);
12902 partial.state.repo = repo.clone();
12903 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12904 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12905 run: Some("exit 1".into()),
12906 ..Default::default()
12907 }];
12908 let selection = runner.state.pr_recovery.clone().unwrap();
12909 partial.state.pr_recovery = Some(selection.clone());
12910 partial.state.pr_head = Some(selection.branch.clone());
12911 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12912 let id = partial.state.id.clone();
12913 let error = match Runner::open_selected(
12914 &repo,
12915 partial.state,
12916 partial.roles,
12917 1,
12918 runner.state.base_commit.clone(),
12919 selection,
12920 )
12921 .await
12922 {
12923 Ok(_) => panic!("setup must fail"),
12924 Err(e) => e,
12925 };
12926 assert_eq!(
12927 error
12928 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12929 .unwrap()
12930 .run,
12931 id
12932 );
12933 let failed = RunState::load(&id).unwrap();
12934 assert_eq!(failed.status, RunStatus::Blocked);
12935 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12936 assert!(Runner::resume(&id).is_err());
12937 runner.state.status = RunStatus::Ready;
12938 runner.state.driver_exited = true;
12939 runner.state.save().unwrap();
12940 let mut retry = runner_at(RunStatus::Reviewing);
12941 retry.state.repo = repo.clone();
12942 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12943 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12944 reselected.approved_at = jiff::Timestamp::now();
12945 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12946 retry.state.pr_head = Some(reselected.branch.clone());
12947 retry.state.pr_recovery = Some(reselected.clone());
12948 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12949 .await
12950 .unwrap();
12951 let restarted = Runner::open_selected(
12952 &repo,
12953 retry.state,
12954 retry.roles,
12955 1,
12956 runner.state.base_commit.clone(),
12957 reselected,
12958 )
12959 .await
12960 .unwrap();
12961 assert_eq!(
12962 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12963 .await
12964 .unwrap(),
12965 head
12966 );
12967 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12968 assert_eq!(
12969 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12970 head
12971 );
12972 }
12973
12974 #[tokio::test]
12975 #[cfg(unix)]
12976 async fn fallback_and_interrupted_restoration_require_coordination() {
12977 use std::os::fd::AsRawFd as _;
12978 ask_test_home();
12979 let tmp = tempfile::tempdir().unwrap();
12980 let mut runner = registered_runner(tmp.path()).await;
12981 runner.state.config.graph.timeout_resource_wait = 1;
12982 let winner = runner.state.candidates[0].clone();
12983 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12984 let leftover = winner.worktree.join("leftover");
12985 std::fs::write(&leftover, "untrusted edits").unwrap();
12986 let next = tmp.path().join("next-fixer");
12987 let mut failed = spec("first");
12988 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12989 let mut successor = spec("second");
12990 successor.command = vec![
12991 "sh".into(),
12992 "-c".into(),
12993 format!("touch '{}'; echo done", next.display()),
12994 ];
12995 let attempts = vec![
12996 (failed.clone(), "first".into()),
12997 (successor.clone(), "second".into()),
12998 ];
12999 let lock_path = runner.state.repo.join(".git/updater.lock");
13000 let external = std::fs::File::create(&lock_path).unwrap();
13001 assert_eq!(
13002 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13003 0
13004 );
13005 let artifacts = tmp.path().join("artifacts");
13006 let build = |spec, seat| SeatJob {
13007 spec,
13008 seat,
13009 cwd: winner.worktree.clone(),
13010 prompt: "fixture".into(),
13011 timeout: Duration::from_secs(10),
13012 allow_write: true,
13013 sessions: false,
13014 artifacts: artifacts.clone(),
13015 stem: "fix".into(),
13016 handover: None,
13017 };
13018 let (_, _, out) = runner
13019 .ask_fixer_among(
13020 attempts.clone(),
13021 "graft",
13022 None,
13023 Some((&winner.worktree, &head)),
13024 build,
13025 )
13026 .await;
13027 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13028 assert!(leftover.exists(), "contention must prevent clean");
13029 assert!(!next.exists());
13030 assert!(runner.state.handovers.is_empty());
13031 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13033 from_run: "earlier".into(),
13034 generation: 1,
13035 mode: crate::recovery::Mode::VerifyRepair,
13036 reasons: Vec::new(),
13037 reviewed_head: None,
13038 rounds: Vec::new(),
13039 open_findings: Vec::new(),
13040 evidence: None,
13041 criteria_inherited: true,
13042 repair: crate::recovery::Repair::Started {
13043 before: head.clone(),
13044 },
13045 });
13046 assert!(
13047 runner
13048 .recovery_repair(&winner)
13049 .await
13050 .unwrap_err()
13051 .is::<crate::repo_lock::ResourceBlocked>()
13052 );
13053 assert!(matches!(
13054 runner.state.recovery.as_ref().unwrap().repair,
13055 crate::recovery::Repair::Started { .. }
13056 ));
13057 assert!(leftover.exists());
13058 drop(external);
13059 runner.recovery_repair(&winner).await.unwrap();
13060 assert!(!leftover.exists());
13061 assert!(matches!(
13062 runner.state.recovery.as_ref().unwrap().repair,
13063 crate::recovery::Repair::Done { .. }
13064 ));
13065 std::fs::remove_file(&lock_path).unwrap();
13067 std::fs::create_dir(&lock_path).unwrap();
13068 std::fs::write(&leftover, "untrusted edits").unwrap();
13069 let (_, _, out) = runner
13070 .ask_fixer_among(
13071 attempts,
13072 "graft",
13073 None,
13074 Some((&winner.worktree, &head)),
13075 build,
13076 )
13077 .await;
13078 assert!(matches!(out, AgentOutcome::Failed(_)));
13079 assert!(leftover.exists());
13080 assert!(!next.exists());
13081 assert_eq!(
13082 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13083 head
13084 );
13085 }
13086
13087 #[cfg(unix)]
13088 async fn registered_runner(root: &Path) -> Runner {
13089 ask_test_home();
13090 let repo = root.join("repo");
13091 std::fs::create_dir_all(&repo).unwrap();
13092 init_repo(&repo);
13093 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13094 let mut runner = runner_at(RunStatus::Prep);
13095 runner.state.repo = repo.clone();
13096 runner.state.config.coordination = crate::config::test_coordination();
13097 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13098 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13099 runner.state.config.graph.advise = false;
13100 runner.roles.implementers = vec![spec("implement")];
13101 runner.roles.judges = vec![spec("judge")];
13102 runner.prep().await.unwrap();
13103 runner
13104 }
13105
13106 #[tokio::test]
13107 #[cfg(unix)]
13108 async fn parked_implementer_persists_completed_work() {
13109 for registered in [false, true] {
13110 let tmp = tempfile::tempdir().unwrap();
13111 let mut runner = registered_runner(tmp.path()).await;
13112 if !registered {
13113 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13114 }
13115 let marker = tmp.path().join("started");
13116 let release = tmp.path().join("release");
13117 let mut implementer = spec("implement");
13118 implementer.command = vec![
13119 "sh".into(),
13120 "-c".into(),
13121 format!(
13122 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13123 marker.display(),
13124 release.display()
13125 ),
13126 ];
13127 runner.roles.implementers = vec![implementer.clone()];
13128 runner.state.config.agents = vec![implementer];
13129 let pause = runner.pause.clone();
13130 let (result, ()) = tokio::join!(runner.implement(), async {
13131 tokio::time::timeout(Duration::from_secs(10), async {
13132 while !marker.exists() {
13133 tokio::time::sleep(Duration::from_millis(10)).await;
13134 }
13135 })
13136 .await
13137 .unwrap();
13138 pause.park();
13139 std::fs::write(&release, "go").unwrap();
13140 });
13141 result.unwrap();
13142 let candidate = &runner.state.candidates[0];
13143 assert!(candidate.commits > 0, "{candidate:?}");
13144 assert!(candidate.summary.contains("saved-result"));
13145 assert_eq!(candidate.files, 1);
13146 let before = candidate.commits;
13147 runner.implement().await.unwrap();
13148 assert_eq!(runner.state.candidates[0].commits, before);
13149 }
13150 }
13151
13152 #[tokio::test]
13153 #[cfg(unix)]
13154 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13155 let tmp = tempfile::tempdir().unwrap();
13156 let mut runner = registered_runner(tmp.path()).await;
13157 let winner = runner.state.candidates[0].clone();
13158 let cache = tmp.path().join("cache");
13159 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13160 runner.state.config.graph.timeout_resource_wait = 0;
13161 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13162 let crate::cache::AcquireOutcome::Acquired(held) =
13163 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13164 else {
13165 panic!("cache must be free")
13166 };
13167 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13168 from_run: "earlier".into(),
13169 generation: 1,
13170 mode: crate::recovery::Mode::VerifyRepair,
13171 reasons: Vec::new(),
13172 reviewed_head: None,
13173 rounds: Vec::new(),
13174 open_findings: Vec::new(),
13175 evidence: None,
13176 criteria_inherited: true,
13177 repair: crate::recovery::Repair::NotStarted,
13178 });
13179 runner.recovery_repair(&winner).await.unwrap();
13180 assert!(matches!(
13181 runner.state.recovery.as_ref().unwrap().repair,
13182 crate::recovery::Repair::WaitingForRestore { .. }
13183 ));
13184 drop(held);
13185 runner.state.resource_block = None;
13186 runner.state.config.graph.timeout_resource_wait = 1;
13187 runner.recovery_repair(&winner).await.unwrap();
13188 assert!(matches!(
13189 runner.state.recovery.as_ref().unwrap().repair,
13190 crate::recovery::Repair::Done { .. }
13191 ));
13192 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13193 }
13194
13195 #[tokio::test]
13196 #[cfg(unix)]
13197 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13198 use std::os::fd::AsRawFd as _;
13199 for graft in [false, true] {
13200 let tmp = tempfile::tempdir().unwrap();
13201 let mut runner = registered_runner(tmp.path()).await;
13202 runner.state.config.graph.timeout_resource_wait = 1;
13203 let winner = runner.state.candidates[0].clone();
13204 let marker = tmp.path().join("fixer-started");
13205 let mut fixer = spec("fixer");
13206 fixer.command = vec![
13207 "sh".into(),
13208 "-c".into(),
13209 format!("echo started >> '{}'; echo done", marker.display()),
13210 ];
13211 runner.roles.fixer = Some(vec![fixer]);
13212 if graft {
13213 let mut loser = winner.clone();
13214 loser.label = 'B';
13215 loser.index = 1;
13216 runner.state.candidates.push(loser);
13217 runner.state.config.graph.graft_runner_up = true;
13218 runner.state.tally = Some(Tally {
13219 first_choice: BTreeMap::new(),
13220 borda: BTreeMap::new(),
13221 winner: winner.label,
13222 rankings: 1,
13223 unanimous_initial: true,
13224 deliberated: false,
13225 changed_votes: 0,
13226 unanimous_final: true,
13227 tie_break: None,
13228 judges: 1,
13229 present: 1,
13230 quorum: 1,
13231 met_quorum: true,
13232 uncontested: None,
13233 });
13234 runner.state.judgements.push(crate::run::Judgement {
13235 judge: 1,
13236 seat: "judge-1".into(),
13237 agent: "judge".into(),
13238 ranking: vec![winner.label, 'B'],
13239 reasons: BTreeMap::new(),
13240 confidence: None,
13241 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13242 order: vec![0, 1],
13243 failed: None,
13244 duration_ms: 0,
13245 });
13246 } else {
13247 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13248 from_run: "earlier".into(),
13249 generation: 1,
13250 mode: crate::recovery::Mode::VerifyRepair,
13251 reasons: Vec::new(),
13252 reviewed_head: None,
13253 rounds: Vec::new(),
13254 open_findings: Vec::new(),
13255 evidence: None,
13256 criteria_inherited: true,
13257 repair: crate::recovery::Repair::NotStarted,
13258 });
13259 }
13260 let reader = crate::repo_lock::SeatLease::for_checkout(
13261 &winner.worktree,
13262 false,
13263 crate::config::test_coordination().as_ref(),
13264 )
13265 .await
13266 .unwrap();
13267 let pause = runner.pause.clone();
13268 let (result, ()) = tokio::join!(
13269 async {
13270 if graft {
13271 runner.graft().await
13272 } else {
13273 runner.recovery_repair(&winner).await
13274 }
13275 },
13276 async {
13277 tokio::time::sleep(Duration::from_millis(100)).await;
13278 pause.park();
13279 }
13280 );
13281 result.unwrap();
13282 assert!(!marker.exists(), "a blocked seat must not spawn");
13283 runner.state = RunState::load(&runner.state.id).unwrap();
13285 if graft {
13286 assert_eq!(
13287 runner.state.graft.as_ref().unwrap().outcome,
13288 GraftOutcome::WaitingForRestore
13289 );
13290 } else {
13291 assert!(matches!(
13292 runner.state.recovery.as_ref().unwrap().repair,
13293 crate::recovery::Repair::WaitingForRestore { .. }
13294 ));
13295 }
13296 drop(reader);
13297 runner.pause = Pause::new();
13298 runner.state.resource_block = None;
13299 let external =
13301 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13302 assert_eq!(
13303 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13304 0
13305 );
13306 let leftover = winner.worktree.join("leftover");
13307 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13308 let result = if graft {
13309 runner.graft().await
13310 } else {
13311 runner.recovery_repair(&winner).await
13312 };
13313 assert!(
13314 result
13315 .unwrap_err()
13316 .is::<crate::repo_lock::ResourceBlocked>()
13317 );
13318 assert!(leftover.exists());
13319 assert!(!marker.exists());
13320 drop(external);
13321 runner.state.resource_block = None;
13322 let result = if graft {
13323 runner.graft().await
13324 } else {
13325 runner.recovery_repair(&winner).await
13326 };
13327 result.unwrap();
13328 assert!(!leftover.exists(), "restore must precede retry");
13329 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13330 if graft {
13331 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13332 } else {
13333 assert!(matches!(
13334 runner.state.recovery.as_ref().unwrap().repair,
13335 crate::recovery::Repair::Done { .. }
13336 ));
13337 }
13338 }
13339 }
13340
13341 #[tokio::test]
13342 #[cfg(unix)]
13343 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13344 use std::os::fd::AsRawFd as _;
13345 let tmp = tempfile::tempdir().unwrap();
13346 let mut runner = registered_runner(tmp.path()).await;
13347 runner.state.config.graph.timeout_resource_wait = 1;
13348 let winner = runner.state.candidates[0].clone();
13349 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13350 runner.state.tally = Some(Tally {
13351 first_choice: BTreeMap::new(),
13352 borda: BTreeMap::new(),
13353 winner: winner.label,
13354 rankings: 0,
13355 unanimous_initial: false,
13356 deliberated: false,
13357 changed_votes: 0,
13358 unanimous_final: false,
13359 tie_break: None,
13360 judges: 0,
13361 present: 0,
13362 quorum: 0,
13363 met_quorum: true,
13364 uncontested: Some("fixture".into()),
13365 });
13366 runner.state.graft = Some(GraftRecord {
13367 pre: pre.clone(),
13368 post: None,
13369 lines: None,
13370 limit: None,
13371 agent: None,
13372 outcome: GraftOutcome::Running,
13373 requested: Vec::new(),
13374 adoption: None,
13375 adoption_error: None,
13376 });
13377 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13378 git::commit_all(&winner.worktree, "unfinished graft")
13379 .await
13380 .unwrap();
13381 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13382 let leftover = winner.worktree.join("leftover");
13383 std::fs::write(&leftover, "untrusted edits").unwrap();
13384 let path = runner.state.repo.join(".git/updater.lock");
13385 let external = std::fs::File::create(&path).unwrap();
13386 assert_eq!(
13387 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13388 0
13389 );
13390 assert!(
13391 runner
13392 .graft()
13393 .await
13394 .unwrap_err()
13395 .is::<crate::repo_lock::ResourceBlocked>()
13396 );
13397 assert!(matches!(
13398 runner.state.graft.as_ref().unwrap().outcome,
13399 GraftOutcome::Running
13400 ));
13401 assert_eq!(
13402 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13403 unfinished
13404 );
13405 assert!(leftover.exists());
13406 drop(external);
13407 std::fs::remove_file(&path).unwrap();
13408 std::fs::create_dir(&path).unwrap();
13409 assert!(runner.graft().await.is_err());
13410 assert!(matches!(
13411 runner.state.graft.as_ref().unwrap().outcome,
13412 GraftOutcome::Running
13413 ));
13414 assert_eq!(
13415 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13416 unfinished
13417 );
13418 assert!(leftover.exists());
13419 std::fs::remove_dir(&path).unwrap();
13420 runner.graft().await.unwrap();
13421 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13422 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13423 assert!(!leftover.exists());
13424 }
13425
13426 #[tokio::test]
13427 #[cfg(unix)]
13428 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13429 use std::os::fd::AsRawFd as _;
13430 let tmp = tempfile::tempdir().unwrap();
13431 let mut runner = registered_runner(tmp.path()).await;
13432 runner.state.prep_complete = Some(false);
13433 runner.state.config.graph.timeout_resource_wait = 1;
13434 runner.state.save().unwrap();
13435 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13436 assert_eq!(
13437 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13438 0
13439 );
13440 runner.execute().await.unwrap();
13441 let disk = RunState::load(&runner.state.id).unwrap();
13442 assert!(disk.driver_exited);
13443 assert_eq!(disk.driver_pid, None);
13444 assert_eq!(runner.state.status, RunStatus::Blocked);
13445 assert_eq!(
13446 runner.state.resource_block.as_ref().unwrap().kind,
13447 "repository coordination"
13448 );
13449 assert!(runner.state.prep_failure.is_none());
13450 assert!(runner.state.seats.is_empty());
13451 runner.pause.park();
13452 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13455 .await
13456 .unwrap()
13457 .unwrap();
13458 assert!(runner.state.parked);
13459 assert!(runner.state.prep_failure.is_none());
13460 let disk = RunState::load(&runner.state.id).unwrap();
13461 assert!(disk.driver_exited);
13462 assert_eq!(disk.driver_pid, None);
13463 }
13464
13465 #[tokio::test]
13466 #[cfg(unix)]
13467 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13468 use std::os::fd::AsRawFd as _;
13469 let tmp = tempfile::tempdir().unwrap();
13470 let mut runner = registered_runner(tmp.path()).await;
13471 runner.state.prep_complete = Some(false);
13472 runner.state.config.graph.timeout_resource_wait = 30;
13473 runner.state.save().unwrap();
13474 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13475 assert_eq!(
13476 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13477 0
13478 );
13479 assert!(
13480 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13481 .await
13482 .is_err()
13483 );
13484 let disk = RunState::load(&runner.state.id).unwrap();
13485 assert!(disk.driver_exited);
13486 assert_eq!(disk.driver_pid, None);
13487 assert!(disk.status.resumable());
13488 assert!(runner.state.driver_exited);
13489 assert_eq!(runner.state.driver_pid, None);
13490 runner.state.save().unwrap();
13491 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13492 }
13493
13494 #[tokio::test]
13495 #[cfg(unix)]
13496 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13497 let tmp = tempfile::tempdir().unwrap();
13498 let mut runner = registered_runner(tmp.path()).await;
13499 let winner = runner.state.candidates[0].clone();
13500 let cache = tmp.path().join("cache");
13501 let started = tmp.path().join("pre-gate-started");
13502 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13503 runner.state.config.verify.gate =
13504 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13505 let home = crate::run::home();
13506 let owner =
13507 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13508 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13509 crate::cache::AcquireOutcome::Acquired(g) => g,
13510 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13511 };
13512 let mut gate = Box::pin(runner.run_gate(&winner));
13513 assert!(
13514 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13515 .await
13516 .is_err()
13517 );
13518 assert!(!started.exists(), "pre-gate must wait for the cache too");
13519 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13523 let editor = crate::repo_lock::SeatLease::checkout_with(
13524 &winner.worktree,
13525 true,
13526 crate::config::test_coordination().as_ref(),
13527 &wait,
13528 )
13529 .await
13530 .unwrap();
13531 drop(editor);
13532 guard.release();
13533 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13534 .await
13535 .unwrap()
13536 .unwrap();
13537 drop(gate);
13538 assert!(outcomes.iter().all(CommandOutcome::ok));
13539 assert!(started.exists());
13540 }
13541
13542 #[tokio::test]
13543 #[cfg(unix)]
13544 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13545 use std::os::fd::AsRawFd as _;
13546
13547 ask_test_home();
13548 let tmp = tempfile::tempdir().unwrap();
13549 let repo = tmp.path().join("repo");
13550 std::fs::create_dir_all(&repo).unwrap();
13551 init_repo(&repo);
13552 let common = repo.join(".git");
13553 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13554 let mut runner = runner_at(RunStatus::Prep);
13555 runner.state.repo = repo.clone();
13556 runner.state.config.coordination = crate::config::test_coordination();
13557 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13558 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13559 runner.state.config.graph.advise = false;
13560 runner.roles.implementers = vec![spec("implement")];
13561 runner.roles.judges = vec![spec("judge")];
13562 runner.prep().await.unwrap();
13563 let winner = runner.state.candidates[0].clone();
13564 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13567 let completed = tmp.path().join("seat-completed");
13568 runner.roles.implementers[0].command = vec![
13569 "sh".into(),
13570 "-c".into(),
13571 format!("touch '{}'; echo completed", completed.display()),
13572 ];
13573 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13574 assert_eq!(
13575 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13576 0
13577 );
13578 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13579 let mut implement = Box::pin(runner.implement());
13580 tokio::select! {
13581 result = &mut implement => panic!("rescue must wait: {result:?}"),
13582 result = tokio::time::timeout(Duration::from_secs(10), async {
13583 while !completed.exists() {
13584 tokio::time::sleep(Duration::from_millis(20)).await;
13585 }
13586 }) => result.unwrap(),
13587 }
13588 assert!(
13589 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13590 .await
13591 .is_err()
13592 );
13593 assert_eq!(
13594 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13595 before
13596 );
13597 assert_eq!(
13598 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13599 0
13600 );
13601 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13602 .await
13603 .unwrap()
13604 .unwrap();
13605 drop(implement);
13606 assert_ne!(
13607 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13608 before
13609 );
13610 assert!(git::is_clean(&winner.worktree).await.unwrap());
13611
13612 let started = tmp.path().join("pre-gate-started");
13613 let proceed = tmp.path().join("pre-gate-proceed");
13614 let checked = tmp.path().join("gate-ran");
13615 runner.state.config.verify.pre_gate = vec![format!(
13616 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13617 started.display(),
13618 proceed.display()
13619 )];
13620 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13621 assert_eq!(
13622 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13623 0
13624 );
13625 let mut gate = Box::pin(runner.run_gate(&winner));
13626 assert!(
13627 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13628 .await
13629 .is_err()
13630 );
13631 assert!(!started.exists());
13632 assert!(!checked.exists());
13633 assert!(git::is_clean(&winner.worktree).await.unwrap());
13634 assert_eq!(
13635 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13636 0
13637 );
13638 tokio::select! {
13639 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13640 result = tokio::time::timeout(Duration::from_secs(10), async {
13641 while !started.exists() {
13642 tokio::time::sleep(Duration::from_millis(20)).await;
13643 }
13644 }) => result.unwrap(),
13645 }
13646 assert_eq!(
13647 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13648 -1,
13649 "mechanical edits must hold the real lock"
13650 );
13651 std::fs::write(proceed, "ready").unwrap();
13652 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13653 .await
13654 .unwrap()
13655 .unwrap();
13656 drop(gate);
13657 assert!(outcomes.iter().all(CommandOutcome::ok));
13658 assert!(checked.exists());
13659 assert!(runner.state.pre_gate_commit.is_some());
13660 assert!(git::is_clean(&winner.worktree).await.unwrap());
13661 assert_eq!(
13662 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13663 0,
13664 "gate completion must release coordination"
13665 );
13666 drop(outsider);
13667 std::fs::remove_file(common.join("updater.lock")).unwrap();
13668 std::fs::create_dir(common.join("updater.lock")).unwrap();
13669 std::fs::remove_file(&started).unwrap();
13670 std::fs::remove_file(&checked).unwrap();
13671 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13672 let error = runner.run_gate(&winner).await.unwrap_err();
13673 assert!(
13674 error
13675 .to_string()
13676 .contains("acquire mandatory repository lock")
13677 );
13678 assert!(
13679 !started.exists(),
13680 "failed acquisition must prevent pre-gate"
13681 );
13682 assert!(!checked.exists(), "failed acquisition must prevent gate");
13683 assert!(git::is_clean(&winner.worktree).await.unwrap());
13684 assert_eq!(
13685 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13686 head
13687 );
13688 }
13689
13690 #[tokio::test]
13691 #[cfg(unix)]
13692 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13693 use std::os::fd::AsRawFd as _;
13694 use std::time::Duration;
13695
13696 ask_test_home();
13697 let tmp = tempfile::tempdir().unwrap();
13698 let repo = tmp.path().join("repo");
13699 std::fs::create_dir_all(&repo).unwrap();
13700 init_repo(&repo);
13701 let common = repo.join(".git");
13702 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13703 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13704 assert_eq!(
13705 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13706 0
13707 );
13708 let mut runner = runner_at(RunStatus::Prep);
13709 runner.state.repo = repo.clone();
13710 runner.state.config.coordination = crate::config::test_coordination();
13711 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13712 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13713 runner.state.config.graph.advise = true;
13714 runner.state.config.graph.advisors = 1;
13715 runner.roles.implementers = vec![spec("implement")];
13716 runner.roles.judges = vec![spec("judge")];
13717 let root = runner.state.worktree_root();
13718 let started = tmp.path().join("setup-started");
13719 let proceed = tmp.path().join("setup-proceed");
13720 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13721 run: Some(format!(
13722 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13723 started.display(),
13724 proceed.display()
13725 )),
13726 ..Default::default()
13727 }];
13728 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13729 let config = git::git(&repo, &["config", "--local", "--list"])
13730 .await
13731 .unwrap();
13732 let mut prep = Box::pin(runner.prep());
13733 assert!(
13734 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13735 .await
13736 .is_err()
13737 );
13738 assert!(
13739 !root.exists(),
13740 "contention must prevent even checkout creation"
13741 );
13742 assert!(!started.exists(), "contention must prevent setup");
13743 assert_eq!(
13744 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13745 branches
13746 );
13747 assert_eq!(
13748 git::git(&repo, &["config", "--local", "--list"])
13749 .await
13750 .unwrap(),
13751 config
13752 );
13753 drop(lock);
13754 tokio::select! {
13755 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13756 result = tokio::time::timeout(Duration::from_secs(10), async {
13757 while !started.exists() {
13758 tokio::time::sleep(Duration::from_millis(20)).await;
13759 }
13760 }) => result.unwrap(),
13761 }
13762 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13763 assert_eq!(
13764 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13765 -1
13766 );
13767 assert!(
13768 tokio::time::timeout(
13769 Duration::from_millis(100),
13770 crate::repo_lock::SeatLease::for_checkout(
13771 &repo,
13772 true,
13773 crate::config::test_coordination().as_ref()
13774 )
13775 )
13776 .await
13777 .is_err(),
13778 "preparation must also exclude local editors"
13779 );
13780 std::fs::write(proceed, "ready").unwrap();
13781 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13782 .await
13783 .unwrap()
13784 .unwrap();
13785 drop(prep);
13786 assert_eq!(runner.state.prep_complete, Some(true));
13787 assert!(root.join("judge-1").exists());
13788 assert!(root.join("advisor-1").exists());
13789 assert_eq!(
13790 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13791 0
13792 );
13793 }
13794
13795 #[tokio::test]
13796 #[cfg(unix)]
13797 async fn preparation_lock_failure_prevents_repository_changes() {
13798 ask_test_home();
13799 let tmp = tempfile::tempdir().unwrap();
13800 let repo = tmp.path().join("repo");
13801 std::fs::create_dir_all(&repo).unwrap();
13802 init_repo(&repo);
13803 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13804 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13807 let mut runner = runner_at(RunStatus::Prep);
13808 runner.state.repo = repo.clone();
13809 runner.state.config.coordination = crate::config::test_coordination();
13810 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13811 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13812 runner.roles.implementers = vec![spec("implement")];
13813 let error = runner.prep().await.unwrap_err();
13814 assert!(
13815 error
13816 .to_string()
13817 .contains("acquire mandatory repository lock")
13818 );
13819 assert!(!runner.state.worktree_root().exists());
13820 assert!(runner.state.candidates.is_empty());
13821 assert!(!repo.join(".git/config.worktree").exists());
13822 assert_eq!(
13823 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13824 "* main"
13825 );
13826 }
13827
13828 #[tokio::test]
13829 async fn preparation_records_creation_intent_before_git_can_fail() {
13830 ask_test_home();
13831 let tmp = tempfile::tempdir().unwrap();
13832 let repo = tmp.path().join("repo");
13833 std::fs::create_dir_all(&repo).unwrap();
13834 init_repo(&repo);
13835 let mut runner = runner_at(RunStatus::Prep);
13836 runner.state.repo = repo.clone();
13837 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13838 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13839 runner.state.config.blind.commit_msg_hook = false;
13840 runner.roles.implementers = vec![spec("one")];
13841 let label = blind::assign_labels(1, runner.state.seed)[0];
13842 let branch = runner.state.branch_for(label);
13843 git::git(&repo, &["branch", &branch]).await.unwrap();
13846 assert!(runner.prep().await.is_err());
13847 let saved = RunState::load(&runner.state.id).unwrap();
13848 assert!(saved.candidates.is_empty());
13849 let event = saved
13850 .events
13851 .iter()
13852 .find(|e| e.node == "prep-intent")
13853 .unwrap();
13854 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13855 assert_eq!(intent["branch"], branch);
13856 assert_eq!(intent["start"], runner.state.base_commit);
13857 assert_eq!(
13858 intent["worktree"],
13859 runner
13860 .state
13861 .worktree_root()
13862 .join(format!("cand-{label}"))
13863 .to_string_lossy()
13864 .as_ref()
13865 );
13866 assert_eq!(intent["picks"], serde_json::json!([]));
13867 }
13868
13869 #[tokio::test]
13870 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13871 ask_test_home();
13872 let tmp = tempfile::tempdir().unwrap();
13873 let repo = tmp.path().join("repo");
13874 std::fs::create_dir_all(&repo).unwrap();
13875 init_repo(&repo);
13876 let mut runner = runner_at(RunStatus::Prep);
13877 runner.state.repo = repo.clone();
13878 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13879 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13880 runner.state.sources.pick_commits.push(refs::Input {
13881 token: "missing".into(),
13882 sha: "0".repeat(40),
13883 provenance: refs::Provenance::ExplicitCli,
13884 });
13885 assert!(runner.prep().await.is_err());
13886 assert!(runner.state.candidates.is_empty());
13887 assert!(!runner.state.worktree_root().exists());
13888 }
13889
13890 #[tokio::test]
13891 async fn prose_and_legacy_inference_never_import_into_candidates() {
13892 ask_test_home();
13893 let tmp = tempfile::tempdir().unwrap();
13894 let repo = tmp.path().join("repo");
13895 std::fs::create_dir_all(&repo).unwrap();
13896 init_repo(&repo);
13897 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13898 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13899 .await
13900 .unwrap();
13901 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13902 git::git(&repo, &["add", "."]).await.unwrap();
13903 git::git(&repo, &["commit", "-m", "historical implementation"])
13904 .await
13905 .unwrap();
13906 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13907 git::git(&repo, &["checkout", "main"]).await.unwrap();
13908 for instruction in [
13909 format!("Example: magi/history/A at {source}"),
13910 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13911 format!("Never import magi/history/A or {source}"),
13912 format!("Conversation answer: I mentioned {source} only as background"),
13913 ] {
13914 let mut runner = runner_at(RunStatus::Prep);
13915 runner.state.repo = repo.clone();
13916 runner.state.base_commit = base.clone();
13917 runner.state.instruction = instruction.clone();
13918 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13919 runner.state.config.blind.commit_msg_hook = false;
13920 runner.state.seeds = vec![refs::Seed {
13921 token: "magi/history/A".into(),
13922 sha: source.clone(),
13923 kind: refs::SeedKind::Unmerged,
13924 branch: true,
13925 detail: String::new(),
13926 }];
13927 runner.roles.implementers = vec![spec("one")];
13928 assert_eq!(seeded_instruction(&runner.state), instruction);
13929 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13931 legacy.as_object_mut().unwrap().remove("sources");
13932 runner.state = serde_json::from_value(legacy).unwrap();
13933 assert_eq!(runner.state.sources, refs::Sources::default());
13934 runner.state.save().unwrap();
13935 runner.state = RunState::load(&runner.state.id).unwrap();
13936 runner.prep().await.unwrap();
13937 for candidate in &runner.state.candidates {
13938 assert_eq!(
13939 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13940 base
13941 );
13942 assert!(!candidate.worktree.join("unwanted").exists());
13943 }
13944 }
13945 }
13946
13947 #[tokio::test]
13948 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13949 ask_test_home();
13950 let tmp = tempfile::tempdir().unwrap();
13951 let repo = tmp.path().join("repo");
13952 std::fs::create_dir_all(&repo).unwrap();
13953 init_repo(&repo);
13954 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13955 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13956 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13957 git::git(&repo, &["add", "."]).await.unwrap();
13958 git::git(&repo, &["commit", "-m", "required seed"])
13959 .await
13960 .unwrap();
13961 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13962 git::git(&repo, &["checkout", "main"]).await.unwrap();
13963 let mut runner = runner_at(RunStatus::Prep);
13964 runner.state.repo = repo.clone();
13965 runner.state.base_commit = base.clone();
13966 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13967 runner.state.config.graph.advise = false;
13968 runner.state.config.blind.commit_msg_hook = false;
13969 runner.state.sources =
13970 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13971 .await
13972 .unwrap();
13973 runner.roles.implementers = vec![spec("one"), spec("two")];
13974 runner.roles.judges = vec![spec("judge")];
13975 let label = blind::assign_labels(2, runner.state.seed)[0];
13976 let branch = runner.state.branch_for(label);
13977 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13978 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13979 .await
13980 .unwrap();
13981 runner.state.candidates.push(Candidate {
13982 index: 0,
13983 label,
13984 agent: "one".into(),
13985 branch,
13986 worktree,
13987 summary: String::new(),
13988 stat: String::new(),
13989 files: 0,
13990 commits: 0,
13991 empty: false,
13992 failed: None,
13993 verified_noop: None,
13994 duration_ms: 0,
13995 folded: false,
13996 });
13997 runner.state.save().unwrap();
13998 runner.state = RunState::load(&runner.state.id).unwrap();
14000 assert_eq!(runner.state.prep_complete, Some(false));
14001 runner.prep().await.unwrap();
14002 assert_eq!(runner.state.prep_complete, Some(true));
14003 assert_eq!(runner.state.candidates.len(), 2);
14004 for candidate in &runner.state.candidates {
14005 assert_eq!(
14006 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14007 git::tree_of(&repo, &seed).await.unwrap()
14008 );
14009 assert_eq!(
14010 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14011 "required commit"
14012 );
14013 }
14014 assert!(runner.state.worktree_root().join("judge-1").exists());
14015 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14016 runner.prep().await.unwrap();
14017 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14018
14019 runner.state.prep_complete = Some(false);
14020 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14021 run: Some(format!(
14022 "test -f '{}'",
14023 tmp.path().join("dependency-ready").display()
14024 )),
14025 ..Default::default()
14026 }];
14027 assert!(runner.prep().await.is_err());
14028 assert!(runner.state.prep_failure.is_some());
14029 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14030 runner.state = RunState::load(&runner.state.id).unwrap();
14031 assert!(
14032 runner.prep().await.is_err(),
14033 "correcting the cause alone must not auto-retry"
14034 );
14035 let failed_at = runner.state.events.last().unwrap().at;
14036 assert!(
14037 runner.retry_preparation(failed_at).is_err(),
14038 "old approvals cannot unlock the hold"
14039 );
14040 let mut task = crate::queue::Task::new(
14041 "retry".into(),
14042 "retry setup".into(),
14043 repo.clone(),
14044 crate::queue::Source::Human,
14045 );
14046 task.start(runner.state.id.clone());
14047 task.hold_machine(Some("setup failed".into()));
14048 task.release_operator("cli", None);
14049 runner
14050 .retry_preparation(task.resumption.as_ref().unwrap().at)
14051 .unwrap();
14052 runner.prep().await.unwrap();
14053 assert!(runner.state.prep_failure.is_none());
14054 assert_eq!(runner.state.prep_complete, Some(true));
14055 assert_eq!(runner.state.candidates.len(), 2);
14056 assert!(runner.state.worktree_root().join("judge-1").exists());
14057 for candidate in &runner.state.candidates {
14058 assert_eq!(
14059 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14060 git::tree_of(&repo, &seed).await.unwrap()
14061 );
14062 }
14063 }
14064
14065 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14066 let mut heads = Vec::new();
14067 for candidate in candidates {
14068 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14069 }
14070 heads
14071 }
14072
14073 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14074 ask_test_home();
14075 let tmp = tempfile::tempdir().unwrap();
14076 let repo = tmp.path().join("repo");
14077 std::fs::create_dir(&repo).unwrap();
14078 init_repo(&repo);
14079 let mut runner = runner_at(RunStatus::Reviewing);
14080 runner.state.repo = repo.clone();
14081 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14082 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14083 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14084 let branch = format!("candidate-{label}");
14085 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14086 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14087 .await
14088 .unwrap();
14089 if label == 'B' {
14090 std::fs::write(worktree.join("change"), "winner").unwrap();
14091 git::rescue_commit(&worktree, "candidate change")
14092 .await
14093 .unwrap();
14094 }
14095 runner.state.candidates.push(Candidate {
14096 index,
14097 label,
14098 agent: "alpha".into(),
14099 branch,
14100 worktree,
14101 summary: "completed reply".into(),
14102 stat: String::new(),
14103 files: usize::from(label == 'B'),
14104 commits: usize::from(label == 'B'),
14105 empty: label == 'A',
14106 failed: None,
14107 verified_noop: None,
14108 duration_ms: 1,
14109 folded: false,
14110 });
14111 }
14112 runner.state.judge_skipped = true;
14113 runner.state.tally = Some(Tally {
14114 first_choice: BTreeMap::new(),
14115 borda: BTreeMap::new(),
14116 winner: 'B',
14117 rankings: 0,
14118 unanimous_initial: false,
14119 deliberated: false,
14120 changed_votes: 0,
14121 unanimous_final: false,
14122 tie_break: None,
14123 judges: 0,
14124 present: 0,
14125 quorum: 0,
14126 met_quorum: true,
14127 uncontested: Some("only B produced a change".into()),
14128 });
14129 runner.state.save().unwrap();
14130 (tmp, runner)
14131 }
14132
14133 #[tokio::test]
14134 #[cfg(unix)]
14135 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14136 for (solo, failed_checkpoint) in
14137 [(true, false), (false, false), (true, true), (false, true)]
14138 {
14139 let (_tmp, mut runner) = preservation_resume_fixture().await;
14140 if solo {
14141 runner.state.candidates.retain(|c| c.label == 'A');
14142 }
14143 runner.state.candidates[0].empty = false;
14144 runner.state.candidates[0].summary.clear();
14145 runner.state.tally = None;
14146 runner.state.judge_skipped = false;
14147 runner.state.status = RunStatus::Implementing;
14148 runner.state.config.disk.min_free_bytes = u64::MAX;
14149 let mut implementer = spec("alpha");
14150 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14151 runner.roles.implementers = vec![implementer];
14152 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14153 if failed_checkpoint {
14154 std::fs::create_dir_all(&patch).unwrap();
14155 }
14156 runner.state.save().unwrap();
14157 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14158 runner.state = RunState::load(&runner.state.id).unwrap();
14159 assert_eq!(runner.state.status, RunStatus::Blocked);
14160 assert!(runner.state.interrupted_implementations.contains(&'A'));
14161 assert_eq!(runner.state.candidates[0].commits, 0);
14162 assert!(!runner.state.candidates[0].empty);
14163 assert!(runner.state.candidates[0].failed.is_none());
14164 if failed_checkpoint {
14165 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14166 std::fs::remove_dir(patch).unwrap();
14167 }
14168 runner.state.resource_block = None;
14169 runner.state.config.disk.min_free_bytes = 0;
14170 runner.roles.implementers[0].command = vec![
14171 "sh".into(),
14172 "-c".into(),
14173 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14174 ];
14175 runner.implement().await.unwrap();
14176 assert!(runner.state.interrupted_implementations.is_empty());
14177 assert!(runner.state.candidate_preservation.is_empty());
14178 assert_eq!(runner.state.candidates[0].commits, 1);
14179 assert!(!runner.state.candidates[0].empty);
14180 assert!(
14181 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14182 .unwrap()
14183 .contains("retried empty interrupted seat")
14184 );
14185 if !solo {
14186 assert_eq!(runner.state.candidates[1].commits, 1);
14187 }
14188 }
14189 }
14190
14191 #[tokio::test]
14192 #[cfg(unix)]
14193 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14194 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14195 let (_tmp, mut runner) = preservation_resume_fixture().await;
14196 runner.state.candidates.retain(|c| c.label == 'A');
14197 runner.state.candidates[0].empty = false;
14198 runner.state.candidates[0].summary.clear();
14199 runner.state.tally = None;
14200 runner.state.judge_skipped = false;
14201 runner.state.status = RunStatus::Implementing;
14202 runner.state.config.disk.min_free_bytes = u64::MAX;
14205 let wt = runner.state.candidates[0].worktree.clone();
14206 std::fs::write(
14207 wt.join("interrupted-source"),
14208 "retain implementation work\n",
14209 )
14210 .unwrap();
14211 if committed {
14212 git::rescue_commit(&wt, "implementation checkpoint")
14213 .await
14214 .unwrap();
14215 }
14216 let lock = git::git(
14217 &wt,
14218 &[
14219 "rev-parse",
14220 "--path-format=absolute",
14221 "--git-path",
14222 "index.lock",
14223 ],
14224 )
14225 .await
14226 .unwrap();
14227 if block_rescue {
14228 std::fs::write(&lock, "fixture-owned lock").unwrap();
14229 }
14230 let mut implementer = spec("alpha");
14231 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14232 runner.roles.implementers = vec![implementer];
14233 runner.state.save().unwrap();
14234 let result = runner.implement().await;
14235 assert_eq!(result.is_err(), block_rescue);
14236 assert_eq!(
14237 runner.state.resource_block.as_ref().unwrap().kind,
14238 "disk pressure"
14239 );
14240 assert!(
14241 runner
14242 .state
14243 .events
14244 .iter()
14245 .any(|e| e.message.contains("agent execution interrupted"))
14246 );
14247 assert!(
14248 runner
14249 .state
14250 .dir()
14251 .join("artifacts/candidate-A-recovery/staged.patch")
14252 .exists()
14253 );
14254 if block_rescue {
14255 let saved = RunState::load(&runner.state.id).unwrap();
14256 assert!(saved.candidate_preservation.contains_key(&'A'));
14257 assert!(!saved.candidates[0].empty);
14258 assert_eq!(
14259 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14260 "retain implementation work\n"
14261 );
14262 std::fs::remove_file(lock).unwrap();
14263 runner.state = saved;
14264 } else {
14265 assert_eq!(runner.state.status, RunStatus::Blocked);
14266 assert_eq!(runner.state.candidates[0].commits, 1);
14267 }
14268 runner.roles.implementers.clear();
14271 runner.state.config.disk.min_free_bytes = 0;
14272 runner.state.resource_block = None;
14273 runner.implement().await.unwrap();
14274 assert!(runner.state.candidate_preservation.is_empty());
14275 assert_eq!(runner.state.candidates[0].commits, 1);
14276 assert!(!runner.state.candidates[0].empty);
14277 assert!(git::is_clean(&wt).await.unwrap());
14278 assert!(
14279 crate::run::read_artifact(&runner.state, "cand-A.patch")
14280 .unwrap()
14281 .contains("retain implementation work")
14282 );
14283 }
14284 }
14285
14286 #[tokio::test]
14287 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14288 let (_tmp, mut runner) = preservation_resume_fixture().await;
14289 runner.implement().await.unwrap();
14291 assert!(runner.state.candidate_preservation.is_empty());
14292 assert!(
14293 !runner
14294 .state
14295 .dir()
14296 .join("artifacts/candidate-A-recovery")
14297 .exists()
14298 );
14299 runner.fold_losers().await.unwrap();
14300 assert!(!runner.state.candidates[0].worktree.exists());
14301 for schema in [14, crate::run::SCHEMA] {
14302 runner.state.schema = schema;
14303 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14304 runner.state.save().unwrap();
14305 runner.state = RunState::load(&runner.state.id).unwrap();
14306 runner.implement().await.unwrap();
14307 assert!(runner.state.candidate_preservation.is_empty());
14308 assert!(runner.state.candidates[0].folded);
14309 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14310 }
14311 }
14312
14313 #[tokio::test]
14314 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14315 let (_tmp, mut runner) = preservation_resume_fixture().await;
14316 let wt = runner.state.candidates[0].worktree.clone();
14317 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14318 runner.state.schema = 14;
14319 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14320 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14321 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14322 for _ in 0..2 {
14323 assert!(runner.implement().await.is_err());
14324 assert_eq!(runner.state.status, RunStatus::Blocked);
14325 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14326 assert!(
14327 runner.state.candidates[0].empty,
14328 "old verdict must remain unusable"
14329 );
14330 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14331 assert!(runner.state.judge_skipped);
14332 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14333 assert!(
14334 crate::run::read_artifact(&runner.state, "cand-A.patch")
14335 .unwrap()
14336 .contains("owned edits")
14337 );
14338 assert!(wt.exists());
14339 assert_eq!(
14340 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14341 .await
14342 .unwrap(),
14343 1
14344 );
14345 runner.state = RunState::load(&runner.state.id).unwrap();
14346 }
14347 }
14348
14349 #[tokio::test]
14350 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14351 let (_tmp, mut runner) = preservation_resume_fixture().await;
14352 let wt = runner.state.candidates[0].worktree.clone();
14353 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14354 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14355 let index_lock = PathBuf::from(
14356 git::git(
14357 &wt,
14358 &[
14359 "rev-parse",
14360 "--path-format=absolute",
14361 "--git-path",
14362 "index.lock",
14363 ],
14364 )
14365 .await
14366 .unwrap(),
14367 );
14368 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14369 runner.state.schema = 14;
14370 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14371 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14372 assert!(runner.implement().await.is_err());
14373 assert!(runner.after_implement().is_err());
14374 assert!(runner.fold_losers().await.is_err());
14375 assert_eq!(
14376 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14377 "failed seat edits"
14378 );
14379 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14380 runner.state = RunState::load(&runner.state.id).unwrap();
14381 std::fs::remove_file(index_lock).unwrap();
14382 assert!(runner.implement().await.is_err());
14384 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14385 assert!(runner.state.candidates[0].empty);
14386 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14387 assert!(
14388 crate::run::read_artifact(&runner.state, "cand-A.patch")
14389 .unwrap()
14390 .contains("failed seat edits")
14391 );
14392 assert!(wt.exists());
14393 }
14394
14395 #[tokio::test]
14396 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14397 let (_tmp, mut runner) = preservation_resume_fixture().await;
14398 let repo = runner.state.repo.clone();
14399 let common = git::git(
14400 &repo,
14401 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14402 )
14403 .await
14404 .unwrap();
14405 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14406 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14407 assert!(runner.fold_losers().await.is_err());
14408 assert!(!runner.state.candidates[0].worktree.exists());
14409 assert!(!runner.state.candidates[0].folded);
14410 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14411 runner.state = RunState::load(&runner.state.id).unwrap();
14412 std::fs::remove_file(branch_lock).unwrap();
14413 let judge = runner.state.worktree_root().join("judge-1");
14414 git::worktree_add_detached(&repo, &judge, "main")
14415 .await
14416 .unwrap();
14417 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14418 runner.roles.judges = vec![spec("judge")];
14419 assert!(runner.fold_losers().await.is_err());
14420 runner.state = RunState::load(&runner.state.id).unwrap();
14421 assert!(
14422 runner.state.candidates[0].folded,
14423 "candidate progress saved before judge failure"
14424 );
14425 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14426 assert!(judge.join("uncommitted").exists());
14427 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14428 runner.fold_losers().await.unwrap();
14429 assert!(!judge.exists());
14430 runner.fold_losers().await.unwrap();
14431 }
14432
14433 #[tokio::test]
14434 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14435 let (_tmp, mut runner) = preservation_resume_fixture().await;
14436 runner.state.candidates.retain(|c| c.label == 'B');
14437 runner.state.tally = None;
14438 runner.state.judge_skipped = false;
14439 runner.state.prep_complete = Some(true);
14440 runner.state.config.graph.advise = false;
14441 runner.state.config.graph.selected_implementer = Some("alpha".into());
14442 let wt = runner.state.candidates[0].worktree.clone();
14443 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14444 runner
14445 .state
14446 .candidate_preservation
14447 .insert('B', "awaiting persistence".into());
14448 let index_lock = git::git(
14449 &wt,
14450 &[
14451 "rev-parse",
14452 "--path-format=absolute",
14453 "--git-path",
14454 "index.lock",
14455 ],
14456 )
14457 .await
14458 .unwrap();
14459 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14460 runner.state.save().unwrap();
14461 assert!(runner.execute_graph().await.is_err());
14462 assert_eq!(runner.state.status, RunStatus::Blocked);
14463 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14464 std::fs::remove_file(index_lock).unwrap();
14465 for _ in 0..2 {
14468 runner.state = RunState::load(&runner.state.id).unwrap();
14469 runner.pause.park();
14470 runner.execute_graph().await.unwrap();
14471 assert_eq!(runner.state.status, RunStatus::Implementing);
14472 assert!(runner.state.candidate_preservation.is_empty());
14473 assert_eq!(runner.state.candidates[0].commits, 2);
14474 assert!(runner.state.tally.is_none());
14475 }
14476 runner.implement().await.unwrap();
14477 assert_eq!(runner.state.status, RunStatus::Judging);
14478 runner.state.status = RunStatus::Blocked;
14480 runner.state.candidates[0].failed = Some("implementation failed".into());
14481 runner.state.save().unwrap();
14482 let error = runner.execute_graph().await.unwrap_err();
14483 assert!(
14484 error
14485 .to_string()
14486 .contains("selected implementation is held")
14487 );
14488 }
14489
14490 #[tokio::test]
14491 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14492 let (_tmp, mut runner) = preservation_resume_fixture().await;
14493 let repo = runner.state.repo.clone();
14494 let candidate = runner.state.candidates[0].clone();
14495 let registered = git::worktree_holding(&repo, &candidate.branch)
14498 .await
14499 .unwrap()
14500 .unwrap();
14501 assert_eq!(
14502 registered.canonicalize().unwrap(),
14503 candidate.worktree.canonicalize().unwrap()
14504 );
14505 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14506 assert_eq!(
14507 git::worktree_holding(&repo, &candidate.branch)
14508 .await
14509 .unwrap(),
14510 Some(registered)
14511 );
14512 assert!(
14513 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14514 "the stale registration initially keeps the branch checked out"
14515 );
14516 runner.fold_losers().await.unwrap();
14517 assert!(runner.state.candidates[0].folded);
14518 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14519 assert_eq!(
14520 git::worktree_holding(&repo, &candidate.branch)
14521 .await
14522 .unwrap(),
14523 None
14524 );
14525 runner.state = RunState::load(&runner.state.id).unwrap();
14526 runner.fold_losers().await.unwrap();
14527 }
14528
14529 #[tokio::test]
14530 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14531 let (_tmp, mut runner) = preservation_resume_fixture().await;
14532 let repo = runner.state.repo.clone();
14533 let candidate = runner.state.candidates[0].clone();
14534 let winner = runner.state.candidates[1].clone();
14535 let registered = git::worktree_holding(&repo, &candidate.branch)
14536 .await
14537 .unwrap()
14538 .unwrap();
14539 assert_eq!(
14540 registered.canonicalize().unwrap(),
14541 candidate.worktree.canonicalize().unwrap()
14542 );
14543 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14544 assert_eq!(
14545 git::worktree_holding(&repo, &candidate.branch)
14546 .await
14547 .unwrap(),
14548 Some(registered)
14549 );
14550 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14551
14552 for _ in 0..2 {
14553 fold_run(&mut runner.state, false, &crate::run::home())
14554 .await
14555 .unwrap();
14556 runner.state = RunState::load(&runner.state.id).unwrap();
14557 assert!(runner.state.candidates[0].folded);
14558 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14559 assert_eq!(
14560 git::worktree_holding(&repo, &candidate.branch)
14561 .await
14562 .unwrap(),
14563 None
14564 );
14565 assert!(!runner.state.candidates[1].folded);
14566 assert!(winner.worktree.exists());
14567 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14568 }
14569 }
14570
14571 #[tokio::test]
14572 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14573 ask_test_home();
14574 let tmp = tempfile::tempdir().unwrap();
14575 let repo = tmp.path().join("repo");
14576 std::fs::create_dir(&repo).unwrap();
14577 init_repo(&repo);
14578 let mut runner = runner_at(RunStatus::Implementing);
14579 runner.state.repo = repo.clone();
14580 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14581 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14582 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14583 let branch = format!("candidate-{label}");
14584 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14585 git::worktree_add_branch(&repo, &wt, &branch, "main")
14586 .await
14587 .unwrap();
14588 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14589 git::git(&wt, &["add", "README.md"]).await.unwrap();
14590 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14591 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14592 runner.state.candidates.push(Candidate {
14593 index,
14594 label,
14595 agent: "alpha".to_owned(),
14596 branch,
14597 worktree: wt,
14598 summary: "completed implementation".to_owned(),
14599 stat: String::new(),
14600 files: 0,
14601 commits: 0,
14602 empty: false,
14603 failed: None,
14604 verified_noop: None,
14605 duration_ms: 42,
14606 folded: false,
14607 });
14608 runner
14609 .state
14610 .candidate_preservation
14611 .insert(label, "awaiting persistence".to_owned());
14612 }
14613 let wt = runner.state.candidates[0].worktree.clone();
14614 let index_lock = git::git(
14615 &wt,
14616 &[
14617 "rev-parse",
14618 "--path-format=absolute",
14619 "--git-path",
14620 "index.lock",
14621 ],
14622 )
14623 .await
14624 .unwrap();
14625 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14626 runner.state.save().unwrap();
14627 assert!(runner.persist_candidates().await.is_err());
14628 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14629 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14630 assert_eq!(
14631 runner.state.candidates[1].commits, 1,
14632 "other results are still collected"
14633 );
14634 assert!(
14635 !runner.state.candidates[0].empty,
14636 "failed rescue is not no change"
14637 );
14638 assert_eq!(
14639 runner.state.candidates[0].summary,
14640 "completed implementation"
14641 );
14642 assert!(runner.after_implement().is_err());
14643 let home = tmp.path().join("fold-home");
14644 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14645 assert!(wt.join("new-file").exists());
14646 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14647 assert!(recovery.join("staged.patch").exists());
14648 assert!(recovery.join("unstaged.patch").exists());
14649 assert_eq!(
14650 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14651 "untracked edits\n"
14652 );
14653 runner.state.begin_pass();
14654 runner.state.end_pass_parked(ParkCause::Upgrade {
14655 from: "1.0".into(),
14656 to: Some("1.1".into()),
14657 });
14658 runner.state.save().unwrap();
14659 runner.state = RunState::load(&runner.state.id).unwrap();
14660 assert!(matches!(
14661 runner.state.passes.last().unwrap().end,
14662 Some(crate::run::PassEnd::Parked {
14663 cause: ParkCause::Upgrade { .. }
14664 })
14665 ));
14666 std::fs::remove_file(index_lock).unwrap();
14667 runner.implement().await.unwrap();
14669 assert!(runner.state.candidate_preservation.is_empty());
14670 assert!(
14671 runner
14672 .state
14673 .candidates
14674 .iter()
14675 .all(|c| c.commits == 1 && !c.empty)
14676 );
14677 assert!(git::is_clean(&wt).await.unwrap());
14678 assert!(
14679 crate::run::read_artifact(&runner.state, "cand-A.patch")
14680 .unwrap()
14681 .contains("later tracked edits")
14682 );
14683 }
14684
14685 #[tokio::test]
14686 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14687 ask_test_home();
14688 let tmp = tempfile::tempdir().unwrap();
14689 let repo = tmp.path().join("repo");
14690 std::fs::create_dir(&repo).unwrap();
14691 init_repo(&repo);
14692 let wt = tmp.path().join("candidate");
14693 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14694 .await
14695 .unwrap();
14696 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14697 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14698 assert!(wt.join("uncommitted").exists());
14699 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14700 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14701 let mut runner = runner_at(RunStatus::Implementing);
14702 runner.state.repo = repo.clone();
14703 runner.state.base_commit = "missing-base".to_owned();
14704 runner.state.candidates.push(Candidate {
14705 index: 0,
14706 label: 'A',
14707 agent: "alpha".into(),
14708 branch: "candidate".into(),
14709 worktree: wt.clone(),
14710 summary: "completed".into(),
14711 stat: String::new(),
14712 files: 0,
14713 commits: 0,
14714 empty: false,
14715 failed: None,
14716 verified_noop: None,
14717 duration_ms: 1,
14718 folded: false,
14719 });
14720 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14721 assert!(
14722 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14723 .await
14724 .is_err()
14725 );
14726 assert!(!runner.state.candidates[0].folded);
14727 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14728 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14729 runner
14730 .state
14731 .candidate_preservation
14732 .insert('A', "pending".into());
14733 assert!(runner.persist_candidates().await.is_err());
14734 assert!(!runner.state.candidates[0].empty);
14735 assert!(!runner.state.candidates[0].folded);
14736 assert!(wt.exists());
14737 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14738 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14739 std::fs::create_dir_all(&patch_path).unwrap();
14740 assert!(
14741 runner.persist_candidates().await.is_err(),
14742 "artifact failure stays pending"
14743 );
14744 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14745 assert!(!runner.state.candidates[0].empty);
14746 std::fs::remove_dir(patch_path).unwrap();
14747 runner.persist_candidates().await.unwrap();
14748 assert!(
14749 runner.state.candidates[0].empty,
14750 "only successful inspection confirms no change"
14751 );
14752 }
14753
14754 fn init_repo(dir: &Path) {
14755 let run = |args: &[&str]| {
14756 let out = std::process::Command::new("git")
14757 .args(args)
14758 .current_dir(dir)
14759 .quiet()
14760 .output()
14761 .expect("spawn git");
14762 assert!(
14763 out.status.success(),
14764 "git {args:?} failed: {}",
14765 String::from_utf8_lossy(&out.stderr)
14766 );
14767 };
14768 run(&["init", "-b", "main"]);
14769 run(&["config", "user.name", "magi test"]);
14770 run(&["config", "user.email", "magi@example.com"]);
14771 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14772 run(&["add", "-A"]);
14773 run(&["commit", "-m", "init"]);
14774 }
14775
14776 fn ask_test_home() {
14784 crate::run::pin_test_home();
14785 }
14786
14787 fn runner_at(status: RunStatus) -> Runner {
14790 let mut state = RunState::new(
14791 PathBuf::from("/nonexistent/repo"),
14792 "main".to_owned(),
14793 "deadbeef".to_owned(),
14794 "task".to_owned(),
14795 Config::default(),
14796 );
14797 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14800 state.id = format!(
14801 "20000101-000000-{:04x}",
14802 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14803 );
14804 state.status = status;
14805 Runner {
14806 state,
14807 roles: ResolvedRoles {
14808 implementers: Vec::new(),
14809 judges: Vec::new(),
14810 reviewers: Vec::new(),
14811 fixer: None,
14812 conductor: conductor(),
14813 implementer_roster: Vec::new(),
14814 judge_roster: Vec::new(),
14815 reviewer_roster: Vec::new(),
14816 },
14817 sem: Arc::new(Semaphore::new(1)),
14818 pause: Pause::new(),
14819 interrupt: Pause::new(),
14820 }
14821 }
14822
14823 #[tokio::test]
14824 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14825 crate::run::pin_test_home();
14826 let mut runner = runner_at(RunStatus::Gating);
14827 runner.state.save().unwrap();
14828 let id = runner.state.id.clone();
14829 let passes = runner.state.passes.len();
14830 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14831 let mut claiming = Box::pin(runner.claim_driver());
14832 std::future::poll_fn(|cx| {
14833 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14834 std::task::Poll::Ready(())
14835 })
14836 .await;
14837 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14838 drop(owner);
14839 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14840 .await
14841 .unwrap()
14842 .unwrap();
14843 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14844 assert!(execution.runner.state.resource_block.is_none());
14845 assert_eq!(
14846 execution.runner.state.liveness(false),
14847 crate::run::Liveness::Live
14848 );
14849 drop(execution);
14850 assert!(RunState::load(&id).unwrap().driver_exited);
14851 }
14852
14853 #[tokio::test]
14854 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14855 crate::run::pin_test_home();
14856 let mut runner = runner_at(RunStatus::Gating);
14857 runner.state.save().unwrap();
14858 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14861 assert!(runner.claim_driver().await.is_err());
14862 assert!(runner.state.driver_exited);
14863 assert_eq!(runner.state.driver_pid, None);
14864 let disk = RunState::load(&runner.state.id).unwrap();
14865 assert!(disk.driver_exited);
14866 assert_eq!(disk.driver_pid, None);
14867 assert_eq!(disk.status, RunStatus::Gating);
14868 }
14869
14870 #[tokio::test]
14871 async fn an_early_graph_error_releases_its_driver() {
14872 crate::run::pin_test_home();
14873 let mut runner = runner_at(RunStatus::Blocked);
14874 runner.state.config.graph.selected_implementer = Some("selected".into());
14875 runner.state.save().unwrap();
14876 assert!(runner.execute().await.is_err());
14877 let disk = RunState::load(&runner.state.id).unwrap();
14878 assert!(disk.driver_exited);
14879 assert_eq!(disk.driver_pid, None);
14880 assert_eq!(disk.status, RunStatus::Blocked);
14881 }
14882
14883 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14884 crate::run::MergeRepair {
14885 question: "repair-question".into(),
14886 head: "old-head".into(),
14887 pr: "https://github.com/o/r/pull/42".into(),
14888 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14889 gate: vec![outcome(Some(0), false)],
14890 gate_head: Some("old-head".into()),
14891 gate_fixes: Vec::new(),
14892 pre_gate: Vec::new(),
14893 phase,
14894 failure: None,
14895 }
14896 }
14897
14898 #[tokio::test]
14899 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14900 crate::run::pin_test_home();
14901 let mut runner = runner_at(RunStatus::Landing);
14902 runner.state.merge_repairs.push(merge_repair_request(
14903 crate::run::MergeRepairPhase::Requested,
14904 ));
14905 runner.state.save().unwrap();
14906 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14907 assert!(
14908 runner
14909 .execute()
14910 .await
14911 .unwrap_err()
14912 .to_string()
14913 .contains("already running")
14914 );
14915 assert_eq!(
14916 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14917 crate::run::MergeRepairPhase::Requested
14918 );
14919 drop(claim);
14920 let mut disk = runner.state.clone();
14921 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14922 disk.save().unwrap();
14923 assert!(
14924 runner
14925 .execute()
14926 .await
14927 .unwrap_err()
14928 .to_string()
14929 .contains("decision changed")
14930 );
14931 assert_eq!(
14932 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14933 crate::run::MergeRepairPhase::Complete
14934 );
14935 }
14936
14937 #[test]
14938 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14939 crate::run::pin_test_home();
14940 let mut runner = runner_at(RunStatus::Landing);
14941 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14942 runner.state.reviews = request.reviews.clone();
14943 runner.state.gate = request.gate.clone();
14944 runner.state.gate_ran = true;
14945 runner.state.land_approval = Some(crate::run::LandApproval {
14946 question: "old".into(),
14947 head: "old-head".into(),
14948 });
14949 runner.state.merge_repairs.push(request);
14950 runner.begin_merge_repair_review(0).unwrap();
14951 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14952 assert!(!runner.state.gate_status().ok());
14953 assert!(runner.state.land_approval.is_none());
14954 let disk = RunState::load(&runner.state.id).unwrap();
14955 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14956 assert!(disk.merge_repairs[0].gate[0].ok());
14957 assert_eq!(
14958 disk.merge_repairs[0].phase,
14959 crate::run::MergeRepairPhase::Reviewing
14960 );
14961 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14962 assert!(disk.reviews.is_empty());
14963 }
14964
14965 #[tokio::test]
14966 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14967 crate::run::pin_test_home();
14968 for phase in [
14969 crate::run::MergeRepairPhase::Fixing,
14970 crate::run::MergeRepairPhase::Requested,
14971 ] {
14972 let mut runner = runner_at(RunStatus::Landing);
14973 runner.state.config.graph.land_rounds = 0;
14974 runner.state.merge_repairs.push(merge_repair_request(phase));
14975 runner.state.save().unwrap();
14976 assert!(!runner.repair_merge_review().await.unwrap());
14977 assert_eq!(runner.state.status, RunStatus::Blocked);
14978 assert_eq!(
14979 runner.state.merge_repairs[0].phase,
14980 crate::run::MergeRepairPhase::Failed
14981 );
14982 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14983 assert!(
14984 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14985 "interrupted"
14986 } else {
14987 "budget exhausted"
14988 })
14989 );
14990 assert!(runner.state.merge.is_none());
14991 }
14992 }
14993
14994 #[test]
14998 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14999 crate::run::pin_test_home();
15000 let mut runner = runner_at(RunStatus::Implementing);
15001 let interrupt = Pause::new();
15002 runner.watch_interrupt(interrupt.clone());
15003
15004 interrupt.park_because("task a1b2 asked to run first");
15005
15006 assert!(runner.park_here().expect("park_here"));
15007 assert!(runner.state.parked);
15008 let last = runner.state.events.last().expect("a park event");
15009 assert_eq!(last.node, "park");
15010 assert!(
15011 last.message.contains("task a1b2 asked to run first"),
15012 "expected the interrupt reason in {:?}",
15013 last.message
15014 );
15015 }
15016
15017 #[test]
15020 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15021 crate::run::pin_test_home();
15022 let mut runner = runner_at(RunStatus::Implementing);
15023 runner.state.begin_pass();
15024 let shutdown = Pause::new();
15025 runner.on_pause(shutdown.clone());
15026 let upgrade = ParkCause::Upgrade {
15027 from: "0.121.0".to_owned(),
15028 to: Some("0.121.1".to_owned()),
15029 };
15030 shutdown.park_for(upgrade.clone());
15031 shutdown.park_for(ParkCause::Operator);
15032
15033 assert!(runner.park_here().expect("park_here"));
15034 assert_eq!(
15035 runner.state.passes[0].end,
15036 Some(crate::run::PassEnd::Parked { cause: upgrade })
15037 );
15038 let last = runner.state.events.last().expect("a park event");
15039 assert!(last.message.contains("0.121.0"), "{}", last.message);
15040 }
15041
15042 #[tokio::test]
15043 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15044 crate::run::pin_test_home();
15045 let mut runner = runner_at(RunStatus::Reviewing);
15046 runner.state.save().unwrap();
15047 let mut execution = runner.claim_driver().await.expect("first execution");
15048 assert_eq!(execution.runner.state.passes.len(), 1);
15049 execution
15050 .driver
15051 .finish(&mut execution.runner.state)
15052 .await
15053 .unwrap();
15054 }
15055
15056 #[tokio::test]
15057 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15058 crate::run::pin_test_home();
15059 let mut runner = runner_at(RunStatus::Reviewing);
15060 runner.state.config.agents = vec![conductor()];
15061 runner.state.config.graph.timeout_resource_wait = 0;
15062 runner.state.save().unwrap();
15063 let id = runner.state.id.clone();
15064 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15065 assert!(runner.claim_driver().await.is_err());
15066 drop(owner);
15067 drop(runner);
15068 let mut resumed = Runner::resume(&id).unwrap();
15069 assert_eq!(resumed.state.driver_claimed, Some(false));
15070 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15071 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15072 let mut execution = resumed
15073 .claim_driver()
15074 .await
15075 .expect("retry initial claim after reload");
15076 assert_eq!(execution.runner.state.driver_claimed, Some(true));
15077 assert_eq!(execution.runner.state.passes.len(), 1);
15078 execution
15079 .driver
15080 .finish(&mut execution.runner.state)
15081 .await
15082 .unwrap();
15083 }
15084
15085 #[tokio::test]
15086 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15087 crate::run::pin_test_home();
15088 let mut runner = runner_at(RunStatus::Reviewing);
15089 runner.state.config.agents = vec![conductor()];
15090 runner.state.save().unwrap();
15091 let id = runner.state.id.clone();
15092 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15093 let mut resumed = Runner::resume(&id).unwrap();
15094 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15095 assert!(resumed.claim_driver().await.is_err());
15096 assert!(RunState::load(&id).unwrap().passes.is_empty());
15097 drop(lease);
15098 let mut execution = resumed.claim_driver().await.unwrap();
15099 execution
15100 .driver
15101 .finish(&mut execution.runner.state)
15102 .await
15103 .unwrap();
15104 }
15105
15106 #[tokio::test]
15107 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15108 crate::run::pin_test_home();
15109 let mut runner = runner_at(RunStatus::Reviewing);
15110 let state = &mut runner.state;
15111 state.config.agents = vec![conductor()];
15112 state.driver_claimed = None;
15113 state.driver_pid = Some(std::process::id());
15114 state.driver_started_at = None;
15115 state.begin_pass();
15116 state.save().unwrap();
15117 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15118 std::fs::write(
15119 state.dir().join("driver.json"),
15120 serde_json::to_vec(&serde_json::json!({
15121 "pid": state.driver_pid, "started_at": null,
15122 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15123 }))
15124 .unwrap(),
15125 )
15126 .unwrap();
15127 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15128 drop(lease);
15129 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15130 let mut resumed = Runner::resume(&state.id).unwrap();
15131 assert!(!resumed.state.driver_exited);
15132 let mut execution = resumed
15133 .claim_driver()
15134 .await
15135 .expect("lease proves previous execution ended");
15136 assert_eq!(execution.runner.state.passes.len(), 2);
15137 execution
15138 .driver
15139 .finish(&mut execution.runner.state)
15140 .await
15141 .unwrap();
15142 }
15143
15144 #[tokio::test]
15145 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15146 crate::run::pin_test_home();
15147 let mut runner = runner_at(RunStatus::Reviewing);
15148 let mut json = serde_json::to_value(&runner.state).unwrap();
15149 json.as_object_mut().unwrap().remove("driver_claimed");
15150 runner.state = serde_json::from_value(json).unwrap();
15151 assert_eq!(runner.state.driver_claimed, None);
15152 runner.state.save().unwrap();
15153 let mut execution = runner
15154 .claim_driver()
15155 .await
15156 .expect("the execution lease, not a missing marker, excludes a second driver");
15157 assert_eq!(execution.runner.state.passes.len(), 1);
15158 execution
15159 .driver
15160 .finish(&mut execution.runner.state)
15161 .await
15162 .unwrap();
15163 }
15164
15165 #[tokio::test]
15166 async fn driver_admission_refuses_a_proven_live_driver() {
15167 crate::run::pin_test_home();
15168 let mut runner = runner_at(RunStatus::Reviewing);
15169 let pid = std::process::id();
15170 runner.state.driver_pid = Some(pid);
15171 runner.state.driver_started_at = crate::proc::process_started_at(pid);
15172 assert!(runner.state.driver_started_at.is_some());
15173 runner.state.save().unwrap();
15174 let error = runner
15175 .claim_driver()
15176 .await
15177 .err()
15178 .expect("a live driver must refuse");
15179 assert!(error.to_string().contains("live driver"), "{error:#}");
15180 let saved = RunState::load(&runner.state.id).unwrap();
15181 assert_eq!(saved.driver_pid, Some(pid));
15182 assert_eq!(saved.status, RunStatus::Reviewing);
15183 assert!(saved.passes.is_empty());
15184 }
15185
15186 #[test]
15187 fn parking_resource_halt_persists_the_pass_once() {
15188 crate::run::pin_test_home();
15189 let mut runner = runner_at(RunStatus::Reviewing);
15190 runner.state.begin_pass();
15191 runner.state.resource_block = Some(crate::run::ResourceBlock {
15192 node: "fix".into(),
15193 kind: "shared build cache".into(),
15194 detail: "cancelled".into(),
15195 at: Timestamp::now(),
15196 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15197 });
15198 let cause = ParkCause::Upgrade {
15199 from: "old".into(),
15200 to: Some("new".into()),
15201 };
15202 runner.pause.park_for(cause.clone());
15203 runner.stop_for_resource().unwrap();
15206 assert!(runner.resource_halt().unwrap());
15207 assert!(runner.park_here().unwrap());
15208 let saved = RunState::load(&runner.state.id).unwrap();
15209 assert!(saved.parked);
15210 assert_eq!(saved.status, RunStatus::Reviewing);
15211 assert!(saved.passes[0].ended_at.is_some());
15212 assert_eq!(
15213 saved.passes[0].end,
15214 Some(crate::run::PassEnd::Parked {
15215 cause: cause.clone()
15216 })
15217 );
15218 assert_eq!(
15219 saved.resource_block.unwrap().cause,
15220 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15221 );
15222 assert_eq!(
15223 saved
15224 .events
15225 .iter()
15226 .filter(|event| event.node == "park")
15227 .count(),
15228 1
15229 );
15230 }
15231
15232 #[test]
15240 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15241 crate::run::pin_test_home();
15242 let mut runner = runner_at(RunStatus::Implementing);
15243 let shutdown = Pause::new();
15244 runner.on_pause(shutdown.clone());
15245 let interrupt = Pause::new();
15246 runner.watch_interrupt(interrupt.clone());
15247
15248 assert!(!runner.park_here().expect("park_here"));
15250 assert!(!runner.state.parked);
15251
15252 interrupt.park_because("test");
15254 assert!(!shutdown.parked());
15255 assert!(runner.park_here().expect("park_here"));
15256 }
15257
15258 #[tokio::test]
15272 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15273 crate::run::pin_test_home();
15274 let mut runner = runner_at(RunStatus::Implementing);
15275 let interrupt = Pause::new();
15276 runner.watch_interrupt(interrupt.clone());
15277
15278 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15279 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15280
15281 let node = async move {
15285 started_tx.send(()).expect("send started");
15286 finish_rx.await.expect("recv finish");
15287 "node finished"
15288 };
15289
15290 let interrupter = async move {
15291 started_rx.await.expect("recv started");
15292 interrupt.park_because("higher-priority task waiting");
15294 tokio::task::yield_now().await;
15298 finish_tx.send(()).expect("send finish");
15299 };
15300
15301 let (node_result, ()) = tokio::join!(node, interrupter);
15302 assert_eq!(
15303 node_result, "node finished",
15304 "the in-flight call ran to completion"
15305 );
15306
15307 assert!(runner.park_here().expect("park_here"));
15310 assert!(runner.state.parked);
15311 }
15312
15313 #[test]
15319 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15320 crate::run::pin_test_home();
15321 let mut runner = runner_at(RunStatus::Judging);
15322 runner.state.config.agents = vec![conductor()];
15326 runner.state.candidates = vec![Candidate {
15327 index: 0,
15328 label: 'A',
15329 agent: "alpha".to_owned(),
15330 branch: "magi/x/A".to_owned(),
15331 worktree: PathBuf::from("/nonexistent/worktree"),
15332 summary: "did the thing".to_owned(),
15333 stat: "1 file changed".to_owned(),
15334 files: 1,
15335 commits: 1,
15336 empty: false,
15337 failed: None,
15338 verified_noop: None,
15339 duration_ms: 1234,
15340 folded: false,
15341 }];
15342 let run_id = runner.state.id.clone();
15343
15344 let interrupt = Pause::new();
15345 runner.watch_interrupt(interrupt.clone());
15346 interrupt.park_because("task c3d4 asked to run first");
15347 assert!(runner.park_here().expect("park_here"));
15348
15349 let resumed = Runner::resume(&run_id).expect("resume");
15350 assert_eq!(resumed.state.candidates.len(), 1);
15351 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15352 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15353 assert_eq!(resumed.state.status, runner.state.status);
15354 assert!(
15355 resumed.state.parked,
15356 "still parked until `execute` actually walks the graph again"
15357 );
15358 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15359 }
15360
15361 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15363 let mut q = ask::Question::new(
15364 run.to_owned(),
15365 "implement".to_owned(),
15366 "impl-A".to_owned(),
15367 "Which storage backend should the cache use?".to_owned(),
15368 String::new(),
15369 vec!["SQLite".to_owned(), "Redis".to_owned()],
15370 );
15371 store.put(&mut q).unwrap();
15372 q
15373 }
15374
15375 #[test]
15376 fn a_failed_runs_open_question_is_abandoned() {
15377 ask_test_home();
15378 let store = ask::Questions::open();
15379 let mut runner = runner_at(RunStatus::Failed);
15380 let run = runner.state.id.clone();
15381 let q = ask_open_question(&store, &run);
15382
15383 runner.settle_questions();
15384
15385 let back = store.get(&q.id).unwrap();
15386 assert!(
15387 !back.status.open(),
15388 "the seat that asked died with the run; nobody is left to read an answer"
15389 );
15390 assert!(
15391 back.detail.contains(&run) && back.detail.contains("failed"),
15392 "the reason names what the run became, not just that it is gone: {}",
15393 back.detail
15394 );
15395 }
15396
15397 #[test]
15398 fn a_merged_runs_open_question_is_abandoned_too() {
15399 ask_test_home();
15400 let store = ask::Questions::open();
15401 for status in [RunStatus::Merged, RunStatus::Ready] {
15404 let mut runner = runner_at(status);
15405 let run = runner.state.id.clone();
15406 let q = ask_open_question(&store, &run);
15407
15408 runner.settle_questions();
15409
15410 let back = store.get(&q.id).unwrap();
15411 assert!(
15412 !back.status.open(),
15413 "{status:?} run's question must not outlive the run"
15414 );
15415 }
15416 }
15417
15418 #[test]
15419 fn a_still_resumable_runs_open_question_is_left_alone() {
15420 ask_test_home();
15421 let store = ask::Questions::open();
15422 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15428 let mut runner = runner_at(status);
15429 let run = runner.state.id.clone();
15430 let q = ask_open_question(&store, &run);
15431
15432 runner.settle_questions();
15433
15434 let back = store.get(&q.id).unwrap();
15435 assert!(
15436 back.status.open(),
15437 "{status:?} is still alive; the question must still be waiting"
15438 );
15439 }
15440 }
15441
15442 #[test]
15443 fn settle_questions_never_touches_an_already_answered_question() {
15444 ask_test_home();
15445 let store = ask::Questions::open();
15446 let mut runner = runner_at(RunStatus::Failed);
15447 let run = runner.state.id.clone();
15448 let mut q = ask_open_question(&store, &run);
15449 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15450 .unwrap();
15451 store.put(&mut q).unwrap();
15452
15453 runner.settle_questions();
15458 runner.settle_questions();
15459
15460 let back = store.get(&q.id).unwrap();
15461 assert_eq!(
15462 back.status,
15463 ask::QuestionStatus::Answered,
15464 "a real answer is a decision on record, never overwritten by a sweep"
15465 );
15466 }
15467
15468 #[tokio::test]
15479 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15480 crate::run::pin_test_home();
15481 let tmp = tempfile::tempdir().expect("tempdir");
15482 let repo = tmp.path().join("repo");
15483 std::fs::create_dir_all(&repo).unwrap();
15484 init_repo(&repo);
15485
15486 let mut config = Config::default();
15487 config.graph.worktree_root = Some(tmp.path().join("wt"));
15488
15489 let mut state = RunState::new(
15490 repo.clone(),
15491 "main".to_owned(),
15492 "deadbeef".to_owned(),
15493 "task".to_owned(),
15494 config,
15495 );
15496 let root = state.worktree_root();
15497 let wt_a = root.join("cand-A");
15498 let wt_b = root.join("cand-B");
15499 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15500 .await
15501 .expect("worktree A");
15502 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15503 .await
15504 .expect("worktree B");
15505
15506 state.candidates = vec![
15507 Candidate {
15508 index: 0,
15509 label: 'A',
15510 agent: "alpha".to_owned(),
15511 branch: "magi/x/A".to_owned(),
15512 worktree: wt_a.clone(),
15513 summary: String::new(),
15514 stat: String::new(),
15515 files: 0,
15516 commits: 0,
15517 empty: false,
15518 failed: None,
15519 verified_noop: None,
15520 duration_ms: 0,
15521 folded: false,
15522 },
15523 Candidate {
15524 index: 1,
15525 label: 'B',
15526 agent: "beta".to_owned(),
15527 branch: "magi/x/B".to_owned(),
15528 worktree: wt_b.clone(),
15529 summary: String::new(),
15530 stat: String::new(),
15531 files: 0,
15532 commits: 0,
15533 empty: false,
15534 failed: None,
15535 verified_noop: None,
15536 duration_ms: 0,
15537 folded: false,
15538 },
15539 ];
15540 state.tally = Some(Tally {
15541 first_choice: BTreeMap::from([('A', 1)]),
15542 borda: BTreeMap::new(),
15543 winner: 'A',
15544 rankings: 1,
15545 unanimous_initial: true,
15546 deliberated: false,
15547 changed_votes: 0,
15548 unanimous_final: true,
15549 tie_break: None,
15550 judges: 1,
15551 present: 1,
15552 quorum: 1,
15553 met_quorum: true,
15554 uncontested: None,
15555 });
15556 state.status = RunStatus::Ready;
15557
15558 fold_run(&mut state, false, &crate::run::home())
15559 .await
15560 .expect("fold_run");
15561
15562 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15563 assert!(
15564 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15565 "the unmerged winner's branch survives"
15566 );
15567 assert!(
15568 !state.candidates[0].folded,
15569 "the winner is not marked folded"
15570 );
15571
15572 assert!(!wt_b.exists(), "the loser's worktree is removed");
15573 assert!(
15574 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15575 "the loser's branch is removed"
15576 );
15577 assert!(state.candidates[1].folded, "the loser is marked folded");
15578 }
15579
15580 #[tokio::test]
15581 #[cfg(unix)]
15582 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15583 use std::os::fd::AsRawFd as _;
15584 use std::time::Duration;
15585
15586 let tmp = tempfile::tempdir().unwrap();
15587 let repo = tmp.path().join("repo");
15588 std::fs::create_dir_all(&repo).unwrap();
15589 init_repo(&repo);
15590 let common = repo.join(".git");
15591 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15592 let home = tmp.path().join("home");
15593 let mut config = Config::default();
15594 config.coordination = crate::config::test_coordination();
15595 config.graph.worktree_root = Some(tmp.path().join("wt"));
15596 config.graph.timeout_resource_wait = 1;
15597 let mut state = RunState::new(
15598 repo.clone(),
15599 "main".into(),
15600 "deadbeef".into(),
15601 "task".into(),
15602 config,
15603 );
15604 let root = state.worktree_root();
15605 let candidate = root.join("cand-A");
15606 let reviewer = root.join("review-1");
15607 let branch = state.branch_for('A');
15608 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15609 .await
15610 .unwrap();
15611 git::worktree_add_detached(&repo, &reviewer, "main")
15612 .await
15613 .unwrap();
15614 git::acquire_worktree_config(&repo).await.unwrap();
15615 state.enabled_worktree_config = true;
15616 state.status = RunStatus::Ready;
15617 state.candidates.push(Candidate {
15618 index: 0,
15619 label: 'A',
15620 agent: "implement".into(),
15621 branch: branch.clone(),
15622 worktree: candidate.clone(),
15623 summary: String::new(),
15624 stat: String::new(),
15625 files: 0,
15626 commits: 0,
15627 empty: false,
15628 failed: None,
15629 verified_noop: None,
15630 duration_ms: 0,
15631 folded: false,
15632 });
15633 state.save_under(&home).unwrap();
15634 let config_before = std::fs::read(common.join("config")).unwrap();
15635 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15636 .await
15637 .unwrap();
15638 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15641 assert_eq!(
15642 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15643 0
15644 );
15645 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15646 assert!(
15647 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15648 .await
15649 .is_err()
15650 );
15651 assert!(candidate.exists());
15652 assert!(reviewer.exists());
15653 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15654 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15655 assert_eq!(
15656 git::git(&repo, &["worktree", "list", "--porcelain"])
15657 .await
15658 .unwrap(),
15659 worktrees_before
15660 );
15661 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15662 .await
15663 .unwrap()
15664 .unwrap_err();
15665 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15666 drop(fold);
15667 assert!(candidate.exists() && reviewer.exists());
15668 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15669 assert_eq!(
15670 state.resource_block.as_ref().unwrap().kind,
15671 "repository coordination"
15672 );
15673 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15674 drop(external);
15675 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15676 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15677 .await
15678 .unwrap()
15679 .unwrap();
15680 drop(fold);
15681 assert!(removed.contains(&branch));
15682 assert!(!root.exists());
15683 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15684 assert!(state.candidates[0].folded);
15685 assert!(!state.enabled_worktree_config);
15686 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15687 let enabled = git::git_raw(
15688 &repo,
15689 &["config", "--local", "--get", "extensions.worktreeConfig"],
15690 )
15691 .await
15692 .unwrap();
15693 assert!(!enabled.ok(), "the last config reference was released");
15694 }
15695
15696 #[tokio::test]
15699 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15700 let tmp = tempfile::tempdir().expect("tempdir");
15701 let repo = tmp.path().join("repo");
15702 std::fs::create_dir_all(&repo).unwrap();
15703 init_repo(&repo);
15704 let home = tmp.path().join("home");
15705
15706 let mut config = Config::default();
15707 config.graph.worktree_root = Some(tmp.path().join("wt"));
15708 let mut state = RunState::new(
15709 repo.clone(),
15710 "main".to_owned(),
15711 "deadbeef".to_owned(),
15712 "task".to_owned(),
15713 config,
15714 );
15715 git::git(&repo, &["branch", "magi/x/A", "main"])
15717 .await
15718 .expect("branch");
15719 state.candidates = vec![Candidate {
15720 index: 0,
15721 label: 'A',
15722 agent: "alpha".to_owned(),
15723 branch: "magi/x/A".to_owned(),
15724 worktree: state.worktree_root().join("cand-A"),
15725 summary: String::new(),
15726 stat: String::new(),
15727 files: 0,
15728 commits: 0,
15729 empty: false,
15730 failed: None,
15731 verified_noop: None,
15732 duration_ms: 0,
15733 folded: true,
15734 }];
15735 state.released_to = Some("20260901-000000-new1".to_owned());
15736 state.released_branches = vec!["magi/x/A".to_owned()];
15737
15738 fold_run(&mut state, true, &home).await.expect("fold_run");
15739 state.status = RunStatus::Merged;
15740 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15741 state.merge = Some(crate::run::MergeOutcome {
15742 mode: MergeMode::Pr,
15743 ok: true,
15744 detail: "confirmed merged".into(),
15745 empty: false,
15746 });
15747 fold_run(&mut state, true, &home)
15748 .await
15749 .expect("merged fold");
15750
15751 assert!(
15752 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15753 "the handed-over branch survives a fold"
15754 );
15755 }
15756
15757 #[tokio::test]
15761 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15762 let tmp = tempfile::tempdir().expect("tempdir");
15763 let repo = tmp.path().join("repo");
15764 std::fs::create_dir_all(&repo).unwrap();
15765 init_repo(&repo);
15766 let run = |args: &[&str]| {
15767 let out = std::process::Command::new("git")
15768 .quiet()
15769 .args(args)
15770 .current_dir(&repo)
15771 .output()
15772 .expect("spawn git");
15773 assert!(out.status.success(), "git {args:?}");
15774 };
15775 run(&["branch", "magi/x/A"]);
15776 run(&["checkout", "-q", "-b", "magi/x/B"]);
15777 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15778 run(&["add", "-A"]);
15779 run(&["commit", "-q", "-m", "work"]);
15780 run(&["checkout", "-q", "main"]);
15781
15782 let probe = RunState::new(
15786 repo.clone(),
15787 "main".to_owned(),
15788 "deadbeef".to_owned(),
15789 "task".to_owned(),
15790 Config::default(),
15791 );
15792 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15793 let bare = tmp.path().join("origin.git");
15794 let out = std::process::Command::new("git")
15795 .quiet()
15796 .args(["init", "-q", "--bare"])
15797 .arg(&bare)
15798 .output()
15799 .expect("spawn git");
15800 assert!(out.status.success(), "git init --bare");
15801 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15802 run(&["push", "-q", "origin", "main"]);
15803
15804 let mut state = RunState::new(
15805 repo.clone(),
15806 "main".to_owned(),
15807 "deadbeef".to_owned(),
15808 "task".to_owned(),
15809 Config::default(),
15810 );
15811 state.seeds = vec![refs::Seed {
15812 token: "magi/27b2/A".to_owned(),
15813 kind: refs::SeedKind::Unresolved,
15814 sha: String::new(),
15815 branch: true,
15816 detail: "no branch or commit named magi/27b2/A".to_owned(),
15817 }];
15818
15819 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15820 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15821 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15822 let detail = empty_candidate_detail(&state, "main");
15823 assert!(detail.starts_with("empty candidate"), "{detail}");
15824 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15825 assert!(detail.contains("--start-from"), "{detail}");
15826 }
15827
15828 #[tokio::test]
15837 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15838 let tmp = tempfile::tempdir().expect("tempdir");
15839 let repo = tmp.path().join("repo");
15840 std::fs::create_dir_all(&repo).unwrap();
15841 init_repo(&repo);
15842
15843 let mut config = Config::default();
15844 config.merge.mode = MergeMode::Local;
15845
15846 let mut state = RunState::new(
15847 repo.clone(),
15848 "main".to_owned(),
15849 "deadbeef".to_owned(),
15850 "task".to_owned(),
15851 config,
15852 );
15853 state.candidates = vec![Candidate {
15854 index: 0,
15855 label: 'A',
15856 agent: "alpha".to_owned(),
15857 branch: "does-not-exist".to_owned(),
15858 worktree: repo.clone(),
15859 summary: String::new(),
15860 stat: String::new(),
15861 files: 0,
15862 commits: 0,
15863 empty: false,
15864 failed: None,
15865 verified_noop: None,
15866 duration_ms: 0,
15867 folded: false,
15868 }];
15869 state.tally = Some(Tally {
15870 first_choice: BTreeMap::from([('A', 1)]),
15871 borda: BTreeMap::new(),
15872 winner: 'A',
15873 rankings: 1,
15874 unanimous_initial: true,
15875 deliberated: false,
15876 changed_votes: 0,
15877 unanimous_final: true,
15878 tie_break: None,
15879 judges: 0,
15880 present: 0,
15881 quorum: 0,
15882 met_quorum: true,
15883 uncontested: Some("only candidate A produced a change".to_owned()),
15884 });
15885 state.reviews = vec![ReviewRound {
15886 round: 1,
15887 head: "deadbeef".to_owned(),
15888 verified_head: None,
15889 verified_at: None,
15890 reviews: Vec::new(),
15891 e2e: Vec::new(),
15892 fix: None,
15893 blocking: 0,
15894 answered: 0,
15895 expected: 0,
15896 clean: true,
15897 verify_retried: false,
15898 e2e_deferred: false,
15899 e2e_defer_reason: None,
15900 progressed: false,
15901 vote_split: false,
15902 reconsideration: Vec::new(),
15903 verdict: None,
15904 }];
15905 state.gate = vec![CommandOutcome {
15906 command: "test".to_owned(),
15907 code: Some(0),
15908 output_tail: String::new(),
15909 duration_ms: 0,
15910 resource_blocked: false,
15911 }];
15912 state.gate_ran = true;
15913 state.status = RunStatus::Ready;
15918 state.merge = Some(MergeOutcome {
15919 mode: MergeMode::Local,
15920 ok: false,
15921 detail: "already concluded".to_owned(),
15922 empty: false,
15923 });
15924
15925 let mut runner = Runner {
15926 state,
15927 roles: ResolvedRoles {
15928 implementers: Vec::new(),
15929 judges: Vec::new(),
15930 reviewers: Vec::new(),
15931 fixer: None,
15932 conductor: conductor(),
15933 implementer_roster: Vec::new(),
15934 judge_roster: Vec::new(),
15935 reviewer_roster: Vec::new(),
15936 },
15937 sem: Arc::new(Semaphore::new(1)),
15938 pause: Pause::new(),
15939 interrupt: Pause::new(),
15940 };
15941
15942 runner.merge().await.expect("merge");
15943
15944 assert_eq!(
15945 runner.state.status,
15946 RunStatus::Ready,
15947 "a concluded run's status must not change on reentry"
15948 );
15949 assert_eq!(
15950 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15951 Some("already concluded"),
15952 "merge must not run again once the node already recorded an outcome"
15953 );
15954 }
15955
15956 #[tokio::test]
15965 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15966 let tmp = tempfile::tempdir().expect("tempdir");
15967 let repo = tmp.path().join("repo");
15968 std::fs::create_dir_all(&repo).unwrap();
15969 init_repo(&repo);
15970
15971 let mut config = Config::default();
15972 config.merge.mode = MergeMode::Local;
15973
15974 let mut state = RunState::new(
15975 repo.clone(),
15976 "main".to_owned(),
15977 "deadbeef".to_owned(),
15978 "task".to_owned(),
15979 config,
15980 );
15981 state.candidates = vec![Candidate {
15982 index: 0,
15983 label: 'A',
15984 agent: "alpha".to_owned(),
15985 branch: "does-not-exist".to_owned(),
15986 worktree: repo.clone(),
15987 summary: String::new(),
15988 stat: String::new(),
15989 files: 0,
15990 commits: 0,
15991 empty: false,
15992 failed: None,
15993 verified_noop: None,
15994 duration_ms: 0,
15995 folded: false,
15996 }];
15997 state.tally = Some(Tally {
15998 first_choice: BTreeMap::from([('A', 1)]),
15999 borda: BTreeMap::new(),
16000 winner: 'A',
16001 rankings: 1,
16002 unanimous_initial: true,
16003 deliberated: false,
16004 changed_votes: 0,
16005 unanimous_final: true,
16006 tie_break: None,
16007 judges: 0,
16008 present: 0,
16009 quorum: 0,
16010 met_quorum: true,
16011 uncontested: Some("only candidate A produced a change".to_owned()),
16012 });
16013 state.reviews = vec![ReviewRound {
16014 round: 1,
16015 head: "deadbeef".to_owned(),
16016 verified_head: None,
16017 verified_at: None,
16018 reviews: Vec::new(),
16019 e2e: Vec::new(),
16020 fix: None,
16021 blocking: 0,
16022 answered: 0,
16023 expected: 0,
16024 clean: true,
16025 verify_retried: false,
16026 e2e_deferred: false,
16027 e2e_defer_reason: None,
16028 progressed: false,
16029 vote_split: false,
16030 reconsideration: Vec::new(),
16031 verdict: None,
16032 }];
16033 state.gate = Vec::new();
16035 state.gate_ran = false;
16036 state.status = RunStatus::Gating;
16037
16038 let mut runner = Runner {
16039 state,
16040 roles: ResolvedRoles {
16041 implementers: Vec::new(),
16042 judges: Vec::new(),
16043 reviewers: Vec::new(),
16044 fixer: None,
16045 conductor: conductor(),
16046 implementer_roster: Vec::new(),
16047 judge_roster: Vec::new(),
16048 reviewer_roster: Vec::new(),
16049 },
16050 sem: Arc::new(Semaphore::new(1)),
16051 pause: Pause::new(),
16052 interrupt: Pause::new(),
16053 };
16054
16055 runner.merge().await.expect("merge");
16056
16057 assert!(
16058 runner.state.merge.is_none(),
16059 "an empty gate must never be read as a passing one: {:?}",
16060 runner.state.merge
16061 );
16062 }
16063
16064 #[tokio::test]
16071 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16072 let tmp = tempfile::tempdir().expect("tempdir");
16073 let repo = tmp.path().join("repo");
16074 std::fs::create_dir_all(&repo).unwrap();
16075 init_repo(&repo);
16076
16077 let config = Config::default();
16079
16080 let mut state = RunState::new(
16081 repo.clone(),
16082 "main".to_owned(),
16083 "deadbeef".to_owned(),
16084 "task".to_owned(),
16085 config,
16086 );
16087 state.candidates = vec![Candidate {
16088 index: 0,
16089 label: 'A',
16090 agent: "alpha".to_owned(),
16091 branch: "does-not-exist".to_owned(),
16092 worktree: repo.clone(),
16093 summary: String::new(),
16094 stat: String::new(),
16095 files: 0,
16096 commits: 0,
16097 empty: false,
16098 failed: None,
16099 verified_noop: None,
16100 duration_ms: 0,
16101 folded: false,
16102 }];
16103 state.tally = Some(Tally {
16104 first_choice: BTreeMap::from([('A', 1)]),
16105 borda: BTreeMap::new(),
16106 winner: 'A',
16107 rankings: 1,
16108 unanimous_initial: true,
16109 deliberated: false,
16110 changed_votes: 0,
16111 unanimous_final: true,
16112 tie_break: None,
16113 judges: 0,
16114 present: 0,
16115 quorum: 0,
16116 met_quorum: true,
16117 uncontested: Some("only candidate A produced a change".to_owned()),
16118 });
16119 state.reviews = vec![ReviewRound {
16120 round: 1,
16121 head: "deadbeef".to_owned(),
16122 verified_head: None,
16123 verified_at: None,
16124 reviews: Vec::new(),
16125 e2e: Vec::new(),
16126 fix: None,
16127 blocking: 0,
16128 answered: 0,
16129 expected: 0,
16130 clean: true,
16131 verify_retried: false,
16132 e2e_deferred: false,
16133 e2e_defer_reason: None,
16134 progressed: false,
16135 vote_split: false,
16136 reconsideration: Vec::new(),
16137 verdict: None,
16138 }];
16139
16140 let mut runner = Runner {
16141 state,
16142 roles: ResolvedRoles {
16143 implementers: Vec::new(),
16144 judges: Vec::new(),
16145 reviewers: Vec::new(),
16146 fixer: None,
16147 conductor: conductor(),
16148 implementer_roster: Vec::new(),
16149 judge_roster: Vec::new(),
16150 reviewer_roster: Vec::new(),
16151 },
16152 sem: Arc::new(Semaphore::new(1)),
16153 pause: Pause::new(),
16154 interrupt: Pause::new(),
16155 };
16156
16157 runner.gate().await.expect("gate");
16158 assert!(
16159 runner.state.gate_ran,
16160 "zero configured commands is still a real attempt, not an unrun gate"
16161 );
16162 assert!(runner.state.gate.is_empty());
16163 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16164 assert_ne!(
16165 runner.state.status,
16166 RunStatus::Blocked,
16167 "a gate with nothing to check must not read as failed"
16168 );
16169
16170 runner.merge().await.expect("merge");
16171 assert_eq!(
16172 runner.state.status,
16173 RunStatus::Ready,
16174 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16175 );
16176 }
16177
16178 #[tokio::test]
16189 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16190 crate::run::pin_test_home();
16191 let home = crate::run::home();
16192
16193 let tmp = tempfile::tempdir().expect("tempdir");
16194 let repo = tmp.path().join("repo");
16195 std::fs::create_dir_all(&repo).unwrap();
16196 init_repo(&repo);
16197 let cache_dir = tmp.path().join("target");
16200
16201 let mut config = Config::default();
16202 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16203 config.graph.timeout_verify = Some(2);
16206 config.graph.timeout_resource_wait = 2;
16207
16208 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16209 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16210 .expect("no io error acquiring directly")
16211 {
16212 crate::cache::AcquireOutcome::Acquired(g) => g,
16213 crate::cache::AcquireOutcome::Busy(b) => {
16214 panic!("expected the direct acquire to win the lease first: {b:?}")
16215 }
16216 };
16217
16218 let mut state = RunState::new(
16219 repo.clone(),
16220 "main".to_owned(),
16221 "deadbeef".to_owned(),
16222 "task".to_owned(),
16223 config,
16224 );
16225 state.candidates = vec![Candidate {
16226 index: 0,
16227 label: 'A',
16228 agent: "alpha".to_owned(),
16229 branch: "does-not-exist".to_owned(),
16230 worktree: repo.clone(),
16231 summary: String::new(),
16232 stat: String::new(),
16233 files: 0,
16234 commits: 0,
16235 empty: false,
16236 failed: None,
16237 verified_noop: None,
16238 duration_ms: 0,
16239 folded: false,
16240 }];
16241 state.tally = Some(Tally {
16242 first_choice: BTreeMap::from([('A', 1)]),
16243 borda: BTreeMap::new(),
16244 winner: 'A',
16245 rankings: 1,
16246 unanimous_initial: true,
16247 deliberated: false,
16248 changed_votes: 0,
16249 unanimous_final: true,
16250 tie_break: None,
16251 judges: 0,
16252 present: 0,
16253 quorum: 0,
16254 met_quorum: true,
16255 uncontested: Some("only candidate A produced a change".to_owned()),
16256 });
16257 state.reviews = vec![ReviewRound {
16258 round: 1,
16259 head: "deadbeef".to_owned(),
16260 verified_head: None,
16261 verified_at: None,
16262 reviews: Vec::new(),
16263 e2e: Vec::new(),
16264 fix: None,
16265 blocking: 0,
16266 answered: 0,
16267 expected: 0,
16268 clean: true,
16269 verify_retried: false,
16270 e2e_deferred: false,
16271 e2e_defer_reason: None,
16272 progressed: false,
16273 vote_split: false,
16274 reconsideration: Vec::new(),
16275 verdict: None,
16276 }];
16277
16278 let mut runner = Runner {
16279 state,
16280 roles: ResolvedRoles {
16281 implementers: Vec::new(),
16282 judges: Vec::new(),
16283 reviewers: Vec::new(),
16284 fixer: None,
16285 conductor: conductor(),
16286 implementer_roster: Vec::new(),
16287 judge_roster: Vec::new(),
16288 reviewer_roster: Vec::new(),
16289 },
16290 sem: Arc::new(Semaphore::new(1)),
16291 pause: Pause::new(),
16292 interrupt: Pause::new(),
16293 };
16294
16295 let started = std::time::Instant::now();
16296 runner.gate().await.expect("gate");
16297 assert!(
16298 started.elapsed() < Duration::from_secs(1),
16299 "a gate with nothing to run must never wait on a lease it never needed"
16300 );
16301 assert!(
16302 runner.state.gate_ran,
16303 "zero commands is still a real, immediate attempt"
16304 );
16305 assert!(runner.state.gate.is_empty());
16306 assert_ne!(
16307 runner.state.status,
16308 RunStatus::Blocked,
16309 "must not read as resource-blocked on a lease it never asked for"
16310 );
16311 }
16312
16313 #[cfg(unix)]
16316 #[tokio::test]
16317 async fn verification_is_standalone_inside_a_live_worker() {
16318 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
16319 if std::env::var_os(CHILD).is_none() {
16320 let home = tempfile::tempdir().unwrap();
16321 let generation = crate::worker::pin_current(home.path()).unwrap();
16322 let output = std::process::Command::new(std::env::current_exe().unwrap())
16323 .args([
16324 "--exact",
16325 "graph::tests::verification_is_standalone_inside_a_live_worker",
16326 "--nocapture",
16327 ])
16328 .env(CHILD, "1")
16329 .env("MAGI_HOME", home.path())
16330 .env("MAGI_WORKER_HOME", home.path())
16331 .env("MAGI_WORKER_ID", "abcdef")
16332 .env("MAGI_WORKER_GENERATION", &generation.id)
16333 .env("MAGI_WORKER_EXE", &generation.executable)
16334 .env("MAGI_RUN", "verification-parent")
16335 .env("MAGI_NODE", "gate")
16336 .env("MAGI_SEAT", "worker-fixture")
16337 .env("MAGI_PROCESS_OWNER", "fixture-owner")
16338 .env("MAGI_NO_AUTOUPDATE", "0")
16339 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
16340 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
16341 .env(
16342 "VERIFY_EXPECT_TOOLCHAIN",
16343 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
16344 )
16345 .env(
16346 "VERIFY_EXPECT_CACHE",
16347 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
16348 )
16349 .output()
16350 .unwrap();
16351 assert!(
16352 output.status.success()
16353 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16354 "{}\n{}",
16355 String::from_utf8_lossy(&output.stdout),
16356 String::from_utf8_lossy(&output.stderr)
16357 );
16358 return;
16359 }
16360 let home = crate::run::home();
16361 let directory = home.join("workers/jobs/abcdef");
16362 std::fs::create_dir_all(&directory).unwrap();
16363 let job = crate::worker::Job {
16364 id: "abcdef".into(),
16365 sequence: 1,
16366 kind: "task".into(),
16367 key: "task:verification".into(),
16368 payload: serde_json::json!({}),
16369 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
16370 version: Some(env!("CARGO_PKG_VERSION").into()),
16371 status: crate::worker::Status::Running,
16372 result: None,
16373 pid: Some(std::process::id()),
16374 created_at: 0,
16375 finished_at: None,
16376 };
16377 std::fs::write(
16378 directory.join("job.json"),
16379 serde_json::to_vec(&job).unwrap(),
16380 )
16381 .unwrap();
16382 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
16383 let assert_worker = || {
16384 assert_eq!(
16385 crate::worker::current_job(&home).unwrap().unwrap().id,
16386 job.id
16387 );
16388 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
16389 assert_eq!(
16390 crate::worker::helper_executable(&home).unwrap(),
16391 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
16392 );
16393 assert!(
16394 crate::updater::Checker::new(&crate::config::Update {
16395 mode: crate::config::UpdateMode::Notify,
16396 interval: None,
16397 auto_restart: false,
16398 })
16399 .is_none(),
16400 "the driver remains pinned, even while tests are standalone"
16401 );
16402 };
16403 assert_worker();
16404 let repo = home.join("repo");
16405 std::fs::create_dir_all(&repo).unwrap();
16406 init_repo(&repo);
16407 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
16410 test -z "${MAGI_WORKER_GENERATION+x}" &&
16411 test -z "${MAGI_WORKER_HOME+x}" &&
16412 test -z "${MAGI_WORKER_EXE+x}" &&
16413 test "$MAGI_HOME/repo" -ef . &&
16414 test "$MAGI_RUN" = verification-parent &&
16415 test "$MAGI_NODE" = gate &&
16416 test "$MAGI_SEAT" = worker-fixture &&
16417 test -n "$MAGI_PROCESS_OWNER" &&
16418 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
16419 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
16420 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
16421 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
16422 assert!(
16424 !std::process::Command::new("sh")
16425 .args(["-c", probe])
16426 .current_dir(&repo)
16427 .status()
16428 .unwrap()
16429 .success()
16430 );
16431 let unrelated_home = tempfile::tempdir().unwrap();
16432 assert!(
16433 crate::worker::current_job(unrelated_home.path()).is_err(),
16434 "an inherited worker id cannot be resolved in a unit fixture's home"
16435 );
16436 let mut runner = runner_at(RunStatus::Gating);
16437 runner.state.repo = repo.clone();
16438 runner.state.config.disk.min_free_bytes = 0;
16439 runner.state.config.coordination = crate::config::test_coordination();
16440 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
16441 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
16442 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
16443 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
16444 let winner = Candidate {
16445 index: 0,
16446 label: 'A',
16447 agent: "fixture".into(),
16448 branch: "main".into(),
16449 worktree: repo.clone(),
16450 summary: String::new(),
16451 stat: String::new(),
16452 files: 0,
16453 commits: 0,
16454 empty: false,
16455 failed: None,
16456 verified_noop: None,
16457 duration_ms: 0,
16458 folded: false,
16459 };
16460 let (e2e, retried, pids) = run_e2e_with_retry(
16461 &mut runner.state,
16462 &["sh".into(), "-c".into()],
16463 &[format!("{probe} && printf initial")],
16464 &repo,
16465 Duration::from_secs(10),
16466 "initial",
16467 )
16468 .await;
16469 assert!(!retried && pids.is_empty());
16470 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
16471 assert_eq!(e2e[0].output_tail, "initial");
16472 let gate = runner.run_gate(&winner).await.unwrap();
16473 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
16474 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
16477 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
16478 passed: false,
16479 outcomes: Vec::new(),
16480 });
16481 runner.state.save().unwrap();
16482 let mut resumed = RunState::load(&runner.state.id).unwrap();
16483 let wait = crate::repo_lock::WaitPolicy::default();
16484 assert!(matches!(
16485 check_gate_repair_e2e(&mut resumed, &winner, &wait)
16486 .await
16487 .unwrap(),
16488 GateFix::Retry
16489 ));
16490 let saved = RunState::load(&resumed.id).unwrap();
16491 let repair = saved.gate_repair_e2e.unwrap();
16492 assert!(repair.passed);
16493 assert_eq!(repair.outcomes.len(), 1);
16494 assert!(repair.outcomes[0].ok());
16495 let landing = run_landing_gate(&mut resumed, &winner, &wait)
16496 .await
16497 .unwrap();
16498 assert_eq!(landing.len(), 1);
16499 assert!(landing[0].ok(), "{landing:?}");
16500 assert_eq!(
16501 std::fs::read_to_string(repo.join("phases")).unwrap(),
16502 "pre\ngate\ne2e\npre\ngate\n"
16503 );
16504 assert_worker();
16505 }
16506
16507 #[cfg(unix)]
16508 #[tokio::test]
16509 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16510 crate::run::pin_test_home();
16511 let tmp = tempfile::tempdir().unwrap();
16512 let mut config = Config::default();
16513 config.disk.min_free_bytes = 0;
16514 let mut state = RunState::new(
16515 tmp.path().to_owned(),
16516 "main".into(),
16517 "base".into(),
16518 "verification evidence".into(),
16519 config,
16520 );
16521 let (outcomes, pids) = run_commands(
16522 &mut state,
16523 "gate",
16524 "gate",
16525 1,
16526 &["sh".into(), "-c".into()],
16527 &[
16528 "printf before-timeout; exec sleep 30".into(),
16529 "kill -TERM $$".into(),
16530 "exit 7".into(),
16531 "printf success".into(),
16532 ],
16533 tmp.path(),
16534 Duration::from_secs(1),
16535 )
16536 .await;
16537 wait_for_timed_out_children_to_die(&pids).await;
16538 assert_eq!(
16539 pids.len(),
16540 1,
16541 "unleased timeout retains its death-check PID"
16542 );
16543 assert_eq!(outcomes.len(), 4);
16544 assert_eq!(outcomes[0].code, None);
16545 assert!(outcomes[0].output_tail.contains("before-timeout"));
16546 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16547 assert_eq!(outcomes[1].code, None);
16548 assert!(outcomes[1].output_tail.contains("command terminated:"));
16549 assert!(!outcomes[1].output_tail.contains("timed out"));
16550 assert_eq!(outcomes[2].code, Some(7));
16551 assert!(!outcomes[2].output_tail.contains("timed out"));
16552 assert_eq!(outcomes[3].code, Some(0));
16553 assert_eq!(outcomes[3].output_tail, "success");
16554 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16555 }
16556
16557 #[tokio::test]
16567 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16568 crate::run::pin_test_home();
16569
16570 let tmp = tempfile::tempdir().expect("tempdir");
16571 let repo = tmp.path().join("repo");
16572 std::fs::create_dir_all(&repo).unwrap();
16573 init_repo(&repo);
16574
16575 let mut config = Config::default();
16576 config.verify.gate = vec![
16577 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16578 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16579 .to_owned(),
16580 ];
16581
16582 let mut state = RunState::new(
16583 repo.clone(),
16584 "main".to_owned(),
16585 "deadbeef".to_owned(),
16586 "task".to_owned(),
16587 config,
16588 );
16589 let run_id = state.id.clone();
16590 state.candidates = vec![Candidate {
16591 index: 0,
16592 label: 'A',
16593 agent: "alpha".to_owned(),
16594 branch: "does-not-exist".to_owned(),
16595 worktree: repo.clone(),
16596 summary: String::new(),
16597 stat: String::new(),
16598 files: 0,
16599 commits: 0,
16600 empty: false,
16601 failed: None,
16602 verified_noop: None,
16603 duration_ms: 0,
16604 folded: false,
16605 }];
16606 state.tally = Some(Tally {
16607 first_choice: BTreeMap::from([('A', 1)]),
16608 borda: BTreeMap::new(),
16609 winner: 'A',
16610 rankings: 1,
16611 unanimous_initial: true,
16612 deliberated: false,
16613 changed_votes: 0,
16614 unanimous_final: true,
16615 tie_break: None,
16616 judges: 0,
16617 present: 0,
16618 quorum: 0,
16619 met_quorum: true,
16620 uncontested: Some("only candidate A produced a change".to_owned()),
16621 });
16622 state.reviews = vec![ReviewRound {
16623 round: 1,
16624 head: "deadbeef".to_owned(),
16625 verified_head: None,
16626 verified_at: None,
16627 reviews: Vec::new(),
16628 e2e: Vec::new(),
16629 fix: None,
16630 blocking: 0,
16631 answered: 0,
16632 expected: 0,
16633 clean: true,
16634 verify_retried: false,
16635 e2e_deferred: false,
16636 e2e_defer_reason: None,
16637 progressed: false,
16638 vote_split: false,
16639 reconsideration: Vec::new(),
16640 verdict: None,
16641 }];
16642
16643 let mut runner = Runner {
16644 state,
16645 roles: ResolvedRoles {
16646 implementers: Vec::new(),
16647 judges: Vec::new(),
16648 reviewers: Vec::new(),
16649 fixer: None,
16650 conductor: conductor(),
16651 implementer_roster: Vec::new(),
16652 judge_roster: Vec::new(),
16653 reviewer_roster: Vec::new(),
16654 },
16655 sem: Arc::new(Semaphore::new(1)),
16656 pause: Pause::new(),
16657 interrupt: Pause::new(),
16658 };
16659
16660 let started_marker = repo.join("started.marker");
16661 let release_marker = repo.join("release.marker");
16662 let poller = tokio::spawn(async move {
16663 for _ in 0..100 {
16668 if started_marker.exists()
16669 && let Ok(s) = crate::run::RunState::load(&run_id)
16670 && let Some(a) = s.active.get("gate")
16671 {
16672 std::fs::write(&release_marker, b"go").expect("release marker");
16673 return Some(a.clone());
16674 }
16675 tokio::time::sleep(Duration::from_millis(50)).await;
16676 }
16677 None
16678 });
16679
16680 runner.gate().await.expect("gate");
16681 let captured = poller.await.expect("poller task");
16682 let captured = captured.expect(
16683 "the poller never saw a `gate` task entry in run.json while the command was \
16684 still blocked on its own release marker",
16685 );
16686
16687 assert_eq!(captured.task.as_deref(), Some("gate"));
16688 assert_eq!(captured.node, "gate");
16689 assert_eq!(captured.index, Some(1));
16690 assert_eq!(captured.total, Some(1));
16691 assert!(
16692 captured
16693 .command
16694 .as_deref()
16695 .is_some_and(|c| c.contains("started.marker")),
16696 "{captured:?}"
16697 );
16698
16699 assert!(
16700 runner.state.active.is_empty(),
16701 "the entry must be cleared once the command actually finished: {:?}",
16702 runner.state.active
16703 );
16704 assert!(runner.state.gate_ran);
16705 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16706 }
16707
16708 #[tokio::test]
16711 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16712 use crate::verdict::{Finding, ReviewVote, Severity};
16713 crate::run::pin_test_home();
16714 let tmp = tempfile::tempdir().expect("tempdir");
16715 let repo = tmp.path().join("repo");
16716 std::fs::create_dir_all(&repo).unwrap();
16717 init_repo(&repo);
16718
16719 for (severity, vote, expect) in [
16720 (Severity::Major, ReviewVote::Reject, true),
16721 (Severity::Minor, ReviewVote::Reject, false),
16722 (Severity::Major, ReviewVote::Approve, false),
16723 ] {
16724 let mut round = review_round(false, 1, 1, 1, false, true);
16725 round.reviews = vec![ReviewRecord {
16726 reviewer: 1,
16727 agent: "alpha".to_owned(),
16728 summary: String::new(),
16729 findings: vec![Finding {
16730 id: "R1-1-1".to_owned(),
16731 severity,
16732 file: None,
16733 line: None,
16734 title: "t".to_owned(),
16735 detail: String::new(),
16736 }],
16737 vote: Some(vote),
16738 failed: None,
16739 duration_ms: 0,
16740 attempts: 0,
16741 }];
16742 let mut state = RunState::new(
16743 repo.clone(),
16744 "main".to_owned(),
16745 "deadbeef".to_owned(),
16746 "task".to_owned(),
16747 Config::default(),
16748 );
16749 state.reviews = vec![round];
16750 let mut runner = Runner {
16751 state,
16752 roles: ResolvedRoles {
16753 implementers: Vec::new(),
16754 judges: Vec::new(),
16755 reviewers: Vec::new(),
16756 fixer: None,
16757 conductor: conductor(),
16758 implementer_roster: Vec::new(),
16759 judge_roster: Vec::new(),
16760 reviewer_roster: Vec::new(),
16761 },
16762 sem: Arc::new(Semaphore::new(1)),
16763 pause: Pause::new(),
16764 interrupt: Pause::new(),
16765 };
16766 let shell = runner.state.config.shell();
16767 runner
16768 .stop_reviewing("round budget spent", &shell, &repo)
16769 .await
16770 .expect("stop_reviewing");
16771 assert_eq!(runner.state.status, RunStatus::Gating);
16772 assert_eq!(
16773 runner.state.contested_handoff.is_some(),
16774 expect,
16775 "{severity:?} + {vote:?}"
16776 );
16777 }
16778 }
16779
16780 #[tokio::test]
16793 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16794 crate::run::pin_test_home();
16795 let home = crate::run::home();
16796
16797 let tmp = tempfile::tempdir().expect("tempdir");
16798 let repo = tmp.path().join("repo");
16799 std::fs::create_dir_all(&repo).unwrap();
16800 init_repo(&repo);
16801 let head = crate::git::rev_parse(&repo, "HEAD")
16802 .await
16803 .expect("rev-parse");
16804 let cache_dir = tmp.path().join("target");
16807
16808 let mut config = Config::default();
16809 config.verify.e2e = vec![format!(
16810 "CARGO_TARGET_DIR='{}' test -f README.md",
16811 cache_dir.display()
16812 )];
16813 config.graph.review_rounds = 1;
16814 config.graph.timeout_verify = Some(2);
16817 config.graph.timeout_resource_wait = 2;
16818
16819 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16820 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16821 .expect("no io error acquiring directly")
16822 {
16823 crate::cache::AcquireOutcome::Acquired(g) => g,
16824 crate::cache::AcquireOutcome::Busy(b) => {
16825 panic!("expected the direct acquire to win the lease first: {b:?}")
16826 }
16827 };
16828
16829 let mut state = RunState::new(
16830 repo.clone(),
16831 "main".to_owned(),
16832 head.clone(),
16833 "task".to_owned(),
16834 config,
16835 );
16836 state.candidates = vec![Candidate {
16837 index: 0,
16838 label: 'A',
16839 agent: "alpha".to_owned(),
16840 branch: "does-not-exist".to_owned(),
16841 worktree: repo.clone(),
16842 summary: String::new(),
16843 stat: String::new(),
16844 files: 0,
16845 commits: 0,
16846 empty: false,
16847 failed: None,
16848 verified_noop: None,
16849 duration_ms: 0,
16850 folded: false,
16851 }];
16852 state.tally = Some(Tally {
16853 first_choice: BTreeMap::from([('A', 1)]),
16854 borda: BTreeMap::new(),
16855 winner: 'A',
16856 rankings: 1,
16857 unanimous_initial: true,
16858 deliberated: false,
16859 changed_votes: 0,
16860 unanimous_final: true,
16861 tie_break: None,
16862 judges: 0,
16863 present: 0,
16864 quorum: 0,
16865 met_quorum: true,
16866 uncontested: Some("only candidate A produced a change".to_owned()),
16867 });
16868 state.reviews = vec![ReviewRound {
16872 round: 1,
16873 head: head.clone(),
16874 verified_head: None,
16875 verified_at: None,
16876 reviews: Vec::new(),
16877 e2e: Vec::new(),
16878 fix: None,
16879 blocking: 1,
16880 answered: 1,
16881 expected: 1,
16882 clean: false,
16883 verify_retried: false,
16884 e2e_deferred: true,
16885 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16886 progressed: false,
16887 vote_split: false,
16888 reconsideration: Vec::new(),
16889 verdict: None,
16890 }];
16891
16892 let mut runner = Runner {
16893 state,
16894 roles: ResolvedRoles {
16895 implementers: Vec::new(),
16896 judges: Vec::new(),
16897 reviewers: Vec::new(),
16898 fixer: None,
16899 conductor: conductor(),
16900 implementer_roster: Vec::new(),
16901 judge_roster: Vec::new(),
16902 reviewer_roster: Vec::new(),
16903 },
16904 sem: Arc::new(Semaphore::new(1)),
16905 pause: Pause::new(),
16906 interrupt: Pause::new(),
16907 };
16908
16909 let shell = runner.state.config.shell();
16910 runner
16911 .stop_reviewing("round budget spent", &shell, &repo)
16912 .await
16913 .expect("stop_reviewing");
16914
16915 let last = runner.state.reviews.last().expect("round record");
16916 assert_eq!(
16917 last.e2e_status(),
16918 E2eStatus::ResourceBlocked,
16919 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16920 failed: {last:?}"
16921 );
16922 assert_eq!(
16923 last.verified_head.as_deref(),
16924 Some(head.as_str()),
16925 "which commit this attempt targeted is known even though nothing finished checking \
16926 it"
16927 );
16928 let first_attempt_at = last
16929 .verified_at
16930 .expect("when this attempt ran is known too");
16931 assert_ne!(
16932 runner.state.status,
16933 RunStatus::Blocked,
16934 "contention is evidence about the machine, not the patch — it must not settle the \
16935 run as blocked: {:?}",
16936 runner.state.status
16937 );
16938 assert!(
16939 !runner
16940 .state
16941 .events
16942 .iter()
16943 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16944 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16945 runner.state.events
16946 );
16947
16948 runner
16953 .stop_reviewing("round budget spent", &shell, &repo)
16954 .await
16955 .expect("stop_reviewing retry");
16956 assert_eq!(
16957 runner.state.reviews.len(),
16958 1,
16959 "no new round was started: {:?}",
16960 runner.state.reviews
16961 );
16962 let last = runner.state.reviews.last().expect("round record");
16963 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16964 assert!(
16965 last.verified_at.expect("still known") > first_attempt_at,
16966 "a second reentry must be a fresh attempt, not a stale copy of the first"
16967 );
16968 assert_ne!(runner.state.status, RunStatus::Blocked);
16969
16970 held.release();
16971 }
16972
16973 #[cfg(unix)]
16974 #[tokio::test]
16975 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
16976 ask_test_home();
16977 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
16978 let tmp = tempfile::tempdir().unwrap();
16979 let repo = tmp.path().join("repo");
16980 std::fs::create_dir(&repo).unwrap();
16981 init_repo(&repo);
16982 let run_git = |args: &[&str]| {
16983 let out = std::process::Command::new("git")
16984 .args(args)
16985 .current_dir(&repo)
16986 .output()
16987 .unwrap();
16988 assert!(
16989 out.status.success(),
16990 "{args:?}: {}",
16991 String::from_utf8_lossy(&out.stderr)
16992 );
16993 };
16994 let origin = tmp.path().join("origin.git");
16995 run_git(&["init", "--bare", origin.to_str().unwrap()]);
16996 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
16997 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
16998 run_git(&["add", "spec.rs"]);
16999 run_git(&["commit", "-m", "initial spec"]);
17000 run_git(&["push", "origin", "main"]);
17001 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17002 run_git(&["checkout", "-b", "fix/review"]);
17003 std::fs::write(
17004 repo.join("check.rs"),
17005 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17006 )
17007 .unwrap();
17008 run_git(&["add", "check.rs"]);
17009 run_git(&["commit", "-m", "use spec"]);
17010 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17011 run_git(&["checkout", "main"]);
17012 std::fs::write(
17013 repo.join("spec.rs"),
17014 "struct AgentSpec { max_turn_secs: u64 }\n",
17015 )
17016 .unwrap();
17017 run_git(&["add", "spec.rs"]);
17018 run_git(&["commit", "-m", "extend spec"]);
17019 run_git(&["push", "origin", "main"]);
17020 run_git(&["checkout", "fix/review"]);
17021
17022 let mut runner = runner_at(RunStatus::Reviewing);
17023 runner.state.repo = repo.clone();
17024 runner.state.base_commit = base;
17025 runner.state.pr_head = Some("fix/review".into());
17026 runner.state.config.graph.review_rounds = max_rounds;
17027 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17028 runner.state.config.graph.sessions = false;
17029 runner.state.config.disk.min_free_bytes = 0;
17030 runner.state.config.verify.e2e =
17031 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17032 runner.state.candidates = vec![Candidate {
17033 index: 0,
17034 label: 'A',
17035 agent: "fixture".into(),
17036 branch: "fix/review".into(),
17037 worktree: repo.clone(),
17038 summary: String::new(),
17039 stat: String::new(),
17040 files: 1,
17041 commits: 1,
17042 empty: false,
17043 failed: None,
17044 verified_noop: None,
17045 duration_ms: 0,
17046 folded: false,
17047 }];
17048 runner.state.tally = Some(Tally {
17049 first_choice: BTreeMap::new(),
17050 borda: BTreeMap::new(),
17051 winner: 'A',
17052 rankings: 0,
17053 unanimous_initial: true,
17054 deliberated: false,
17055 changed_votes: 0,
17056 unanimous_final: true,
17057 tie_break: None,
17058 judges: 0,
17059 present: 0,
17060 quorum: 0,
17061 met_quorum: true,
17062 uncontested: Some("review-only".into()),
17063 });
17064 let mut last = review_round(false, 0, 1, 1, false, false);
17065 last.head = reviewed.clone();
17066 last.e2e_deferred = true;
17067 last.e2e[0].resource_blocked = true;
17068 last.e2e[0].output_tail = "waiting for the shared build cache".into();
17069 if max_rounds == 1 {
17070 last.e2e.clear();
17071 }
17072 runner.state.reviews = vec![last];
17073 let calls = tmp.path().join("fix-calls");
17074 let mut fixer = spec("repair");
17075 fixer
17076 .env
17077 .insert("FIX_CALLS".into(), calls.display().to_string());
17078 fixer.command = vec!["sh".into(), "-c".into(), r#"
17079 echo fix >> "$FIX_CALLS"
17080 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
17081 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
17082 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
17083 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
17084 "#.into()];
17085 let mut reviewer = spec("reviewer");
17086 reviewer.command = vec![
17087 "sh".into(),
17088 "-c".into(),
17089 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
17090 .into(),
17091 ];
17092 runner.roles.fixer = Some(vec![fixer]);
17093 runner.roles.reviewers = vec![reviewer];
17094
17095 runner.sync_to_base().await.unwrap();
17096 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
17097 assert_ne!(rebased, reviewed);
17098 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
17099 if reentry {
17100 let shell = runner.state.config.shell();
17103 runner
17104 .stop_unresolved_review("deferred verification", &shell, &repo)
17105 .await
17106 .unwrap();
17107 assert!(runner.catchup_repairable());
17108 assert!(!calls.exists());
17109 runner.state = RunState::load(&runner.state.id).unwrap();
17110 assert!(runner.catchup_repairable());
17111 }
17112 runner.review_loop().await.unwrap();
17113 let first = &runner.state.reviews[0];
17114 assert_eq!(first.head, reviewed);
17115 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
17116 assert!(
17117 first.verify_retried,
17118 "a reproduced build failure gets exactly one retry: {:?}",
17119 first.e2e
17120 );
17121 assert!(first.e2e[0].output_tail.contains("E0063"));
17122 assert!(!runner.catchup_repairable());
17123 if max_rounds == 2 {
17124 assert_eq!(
17125 runner.state.status,
17126 RunStatus::Gating,
17127 "{}",
17128 crate::report::run(&runner.state)
17129 );
17130 assert_eq!(runner.state.reviews.len(), 2);
17131 assert!(first.fix.as_ref().unwrap().committed);
17132 assert!(runner.state.reviews[1].clean);
17133 assert_eq!(
17134 runner
17135 .state
17136 .reviews
17137 .iter()
17138 .filter(|r| r.fix.is_some())
17139 .count(),
17140 1
17141 );
17142 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
17143 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
17144 } else {
17145 assert_eq!(runner.state.status, RunStatus::Blocked);
17146 assert!(first.fix.is_none());
17147 assert!(!calls.exists());
17148 let recovery = crate::recovery::assess(
17149 "prior",
17150 Some(&runner.state),
17151 &runner.landing_base(),
17152 crate::recovery::Relation::Diverged,
17153 );
17154 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
17155 let reasons = recovery.reasons.join("\n");
17156 assert!(
17157 reasons.contains(&reviewed) && reasons.contains(&rebased),
17158 "{reasons}"
17159 );
17160 assert!(
17161 reasons.contains("base sync to")
17162 && reasons.contains("additional review is required"),
17163 "{reasons}"
17164 );
17165 }
17166 }
17167 }
17168
17169 #[tokio::test]
17181 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
17182 crate::run::pin_test_home();
17183 let home = crate::run::home();
17184
17185 let tmp = tempfile::tempdir().expect("tempdir");
17186 let repo = tmp.path().join("repo");
17187 std::fs::create_dir_all(&repo).unwrap();
17188 init_repo(&repo);
17189 let head = crate::git::rev_parse(&repo, "HEAD")
17190 .await
17191 .expect("rev-parse");
17192 let cache_dir = tmp.path().join("target");
17193
17194 let mut config = Config::default();
17195 config.verify.e2e = vec![format!(
17196 "CARGO_TARGET_DIR='{}' test -f README.md",
17197 cache_dir.display()
17198 )];
17199 config.graph.review_rounds = 1;
17200 config.graph.timeout_verify = Some(2);
17201 config.graph.timeout_resource_wait = 2;
17202
17203 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17204 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17205 .expect("no io error acquiring directly")
17206 {
17207 crate::cache::AcquireOutcome::Acquired(g) => g,
17208 crate::cache::AcquireOutcome::Busy(b) => {
17209 panic!("expected the direct acquire to win the lease first: {b:?}")
17210 }
17211 };
17212
17213 let mut state = RunState::new(
17214 repo.clone(),
17215 "main".to_owned(),
17216 head.clone(),
17217 "task".to_owned(),
17218 config,
17219 );
17220 state.candidates = vec![Candidate {
17221 index: 0,
17222 label: 'A',
17223 agent: "alpha".to_owned(),
17224 branch: "does-not-exist".to_owned(),
17225 worktree: repo.clone(),
17226 summary: String::new(),
17227 stat: String::new(),
17228 files: 0,
17229 commits: 0,
17230 empty: false,
17231 failed: None,
17232 verified_noop: None,
17233 duration_ms: 0,
17234 folded: false,
17235 }];
17236 state.tally = Some(Tally {
17237 first_choice: BTreeMap::from([('A', 1)]),
17238 borda: BTreeMap::new(),
17239 winner: 'A',
17240 rankings: 1,
17241 unanimous_initial: true,
17242 deliberated: false,
17243 changed_votes: 0,
17244 unanimous_final: true,
17245 tie_break: None,
17246 judges: 0,
17247 present: 0,
17248 quorum: 0,
17249 met_quorum: true,
17250 uncontested: Some("only candidate A produced a change".to_owned()),
17251 });
17252 state.reviews = vec![ReviewRound {
17256 round: 1,
17257 head: head.clone(),
17258 verified_head: Some(head.clone()),
17259 verified_at: Some(jiff::Timestamp::now()),
17260 reviews: Vec::new(),
17261 e2e: vec![CommandOutcome {
17262 command: format!(
17263 "CARGO_TARGET_DIR='{}' test -f README.md",
17264 cache_dir.display()
17265 ),
17266 code: None,
17267 output_tail: "waiting for the shared build cache".to_owned(),
17268 duration_ms: 0,
17269 resource_blocked: true,
17270 }],
17271 fix: None,
17272 blocking: 1,
17273 answered: 1,
17274 expected: 1,
17275 clean: false,
17276 verify_retried: false,
17277 e2e_deferred: false,
17278 e2e_defer_reason: None,
17279 progressed: false,
17280 vote_split: false,
17281 reconsideration: Vec::new(),
17282 verdict: None,
17283 }];
17284
17285 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17286 let mut runner = Runner {
17287 state,
17288 roles: ResolvedRoles {
17289 implementers: Vec::new(),
17290 judges: Vec::new(),
17291 reviewers: Vec::new(),
17292 fixer: None,
17293 conductor: conductor(),
17294 implementer_roster: Vec::new(),
17295 judge_roster: Vec::new(),
17296 reviewer_roster: Vec::new(),
17297 },
17298 sem: Arc::new(Semaphore::new(1)),
17299 pause: Pause::new(),
17300 interrupt: Pause::new(),
17301 };
17302
17303 runner.review_loop().await.expect("review_loop");
17308
17309 assert_eq!(
17310 runner.state.reviews.len(),
17311 1,
17312 "no new round was started on top of the unresolved one: {:?}",
17313 runner.state.reviews
17314 );
17315 let last = &runner.state.reviews[0];
17316 assert_eq!(
17317 last.e2e_status(),
17318 E2eStatus::ResourceBlocked,
17319 "still contended: {last:?}"
17320 );
17321 assert!(
17322 last.verified_at.expect("still known") > first_attempt_at,
17323 "review_loop must have actually retried the check, not left it exactly as found"
17324 );
17325 assert_ne!(
17326 runner.state.status,
17327 RunStatus::Blocked,
17328 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17329 runner.state.status
17330 );
17331
17332 held.release();
17333 }
17334
17335 #[tokio::test]
17336 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17337 crate::run::pin_test_home();
17338 for status in [RunStatus::Blocked, RunStatus::Gating] {
17339 let mut runner = runner_at(status);
17340 runner.state.config.graph.land = true;
17341 runner.state.gate_ran = true;
17342 runner.state.merge = Some(MergeOutcome {
17343 mode: MergeMode::Pr,
17344 ok: false,
17345 detail: "the fixer produced no commit".into(),
17346 empty: false,
17347 });
17348 let url = "https://example.invalid/o/r/pull/1";
17349 runner.state.pr = Some(crate::run::PrRecord {
17350 url: url.into(),
17351 number: 1,
17352 state: "open".into(),
17353 checks: "red".into(),
17354 round: 1,
17355 rounds: 1,
17356 red_at_merge: Vec::new(),
17357 });
17358 runner.state.save().unwrap();
17359 runner.execute_graph().await.unwrap();
17363 assert_eq!(runner.state.status, RunStatus::Blocked);
17364 assert!(runner.state.land_sync_pending);
17365 assert!(runner.state.events.iter().any(|event|
17366 event.node == "land" && event.message == format!("watching {url}")));
17367 assert_eq!(
17368 runner.state.merge.as_ref().unwrap().detail,
17369 "the fixer produced no commit"
17370 );
17371 }
17372 }
17373
17374 #[tokio::test]
17375 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17376 crate::run::pin_test_home();
17377 let tmp = tempfile::tempdir().expect("tempdir");
17378 let repo = tmp.path().join("repo");
17379 std::fs::create_dir_all(&repo).unwrap();
17380 init_repo(&repo);
17381
17382 let mut config = Config::default();
17383 config.merge.mode = MergeMode::Pr;
17384 config.graph.land = true;
17385 config.graph.land_approval = false;
17386
17387 let mut state = RunState::new(
17388 repo.clone(),
17389 "main".to_owned(),
17390 "deadbeef".to_owned(),
17391 "task".to_owned(),
17392 config,
17393 );
17394 state.candidates = vec![Candidate {
17395 index: 0,
17396 label: 'A',
17397 agent: "alpha".to_owned(),
17398 branch: "does-not-exist".to_owned(),
17399 worktree: repo.clone(),
17400 summary: String::new(),
17401 stat: String::new(),
17402 files: 0,
17403 commits: 0,
17404 empty: false,
17405 failed: None,
17406 verified_noop: None,
17407 duration_ms: 0,
17408 folded: false,
17409 }];
17410 state.tally = Some(Tally {
17411 first_choice: BTreeMap::from([('A', 1)]),
17412 borda: BTreeMap::new(),
17413 winner: 'A',
17414 rankings: 1,
17415 unanimous_initial: true,
17416 deliberated: false,
17417 changed_votes: 0,
17418 unanimous_final: true,
17419 tie_break: None,
17420 judges: 0,
17421 present: 0,
17422 quorum: 0,
17423 met_quorum: true,
17424 uncontested: Some("only candidate A produced a change".to_owned()),
17425 });
17426 state.reviews = vec![ReviewRound {
17427 round: 1,
17428 head: "deadbeef".to_owned(),
17429 verified_head: None,
17430 verified_at: None,
17431 reviews: Vec::new(),
17432 e2e: Vec::new(),
17433 fix: None,
17434 blocking: 0,
17435 answered: 0,
17436 expected: 0,
17437 clean: true,
17438 verify_retried: false,
17439 e2e_deferred: false,
17440 e2e_defer_reason: None,
17441 progressed: false,
17442 vote_split: false,
17443 reconsideration: Vec::new(),
17444 verdict: None,
17445 }];
17446 state.gate = vec![CommandOutcome {
17447 command: "test".to_owned(),
17448 code: Some(0),
17449 output_tail: String::new(),
17450 duration_ms: 0,
17451 resource_blocked: false,
17452 }];
17453 state.gate_ran = true;
17454 state.status = RunStatus::Landing;
17458 state.merge = Some(MergeOutcome {
17459 mode: MergeMode::Pr,
17460 ok: true,
17461 detail: "https://example.invalid/x/y/pull/1".to_owned(),
17462 empty: false,
17463 });
17464
17465 ask_test_home();
17469 state.save().expect("save landing run");
17472 let store = ask::Questions::open();
17473 let q = ask_open_question(&store, &state.id);
17474
17475 let mut runner = Runner {
17476 state,
17477 roles: ResolvedRoles {
17478 implementers: Vec::new(),
17479 judges: Vec::new(),
17480 reviewers: Vec::new(),
17481 fixer: None,
17482 conductor: conductor(),
17483 implementer_roster: Vec::new(),
17484 judge_roster: Vec::new(),
17485 reviewer_roster: Vec::new(),
17486 },
17487 sem: Arc::new(Semaphore::new(1)),
17488 pause: Pause::new(),
17489 interrupt: Pause::new(),
17490 };
17491
17492 runner.execute().await.expect("execute");
17497
17498 assert_eq!(
17499 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17500 Some("https://example.invalid/x/y/pull/1"),
17501 "reentry must not push again or open a second pull request over the \
17502 one `land` is already watching"
17503 );
17504 assert_ne!(
17505 runner.state.status,
17506 RunStatus::Landing,
17507 "land could not actually reach the fake pull request, so it must \
17508 have given up rather than left the run silently parked forever"
17509 );
17510 assert_eq!(runner.state.status, RunStatus::Blocked);
17514 assert!(
17515 store.get(&q.id).unwrap().status.open(),
17516 "Blocked is still alive; settle_questions must have been a no-op here"
17517 );
17518 }
17519
17520 fn state_with_round(round: ReviewRound) -> RunState {
17521 let mut s = RunState::new(
17522 PathBuf::from("/repo"),
17523 "main".to_owned(),
17524 "abc1234".to_owned(),
17525 "add retries".to_owned(),
17526 Config::default(),
17527 );
17528 s.reviews = vec![round];
17529 s
17530 }
17531
17532 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17533 crate::verdict::Finding {
17534 id: id.to_owned(),
17535 severity,
17536 file: None,
17537 line: None,
17538 title: title.to_owned(),
17539 detail: String::new(),
17540 }
17541 }
17542
17543 #[test]
17544 fn pr_body_names_open_findings_and_declined_ones() {
17545 let round = ReviewRound {
17546 round: 2,
17547 head: "deadbee".to_owned(),
17548 verified_head: None,
17549 verified_at: None,
17550 reviews: vec![ReviewRecord {
17551 attempts: 0,
17552 reviewer: 1,
17553 agent: "alpha".to_owned(),
17554 summary: String::new(),
17555 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17556 vote: None,
17557 failed: None,
17558 duration_ms: 0,
17559 }],
17560 e2e: vec![CommandOutcome {
17561 command: "cargo test".to_owned(),
17562 code: Some(0),
17563 output_tail: String::new(),
17564 duration_ms: 0,
17565 resource_blocked: false,
17566 }],
17567 verify_retried: false,
17568 e2e_deferred: false,
17569 e2e_defer_reason: None,
17570 fix: Some(FixRecord {
17571 agent: "alpha".to_owned(),
17572 addressed: Vec::new(),
17573 rejected: vec![crate::verdict::Rejection {
17574 id: "R1-1-1".to_owned(),
17575 why: "not reachable from any caller".to_owned(),
17576 }],
17577 notes: String::new(),
17578 committed: true,
17579 failed: None,
17580 duration_ms: 0,
17581 continuation: None,
17582 }),
17583 blocking: 0,
17584 answered: 1,
17585 expected: 1,
17586 clean: false,
17587 progressed: true,
17588 vote_split: false,
17589 reconsideration: Vec::new(),
17590 verdict: None,
17591 };
17592 let state = state_with_round(round);
17593 let body = pr_message(&state, 'A').body;
17594
17595 assert!(body.contains("add retries"), "the task must still be there");
17596 assert!(body.contains("R2-1-1"), "{body}");
17597 assert!(!body.contains("unused import"), "{body}");
17598 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17599 assert!(
17600 !body.contains("not reachable from any caller"),
17601 "the reason it was declined: {body}"
17602 );
17603 }
17604
17605 #[test]
17606 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17607 let round = ReviewRound {
17608 round: 1,
17609 head: "deadbee".to_owned(),
17610 verified_head: None,
17611 verified_at: None,
17612 reviews: vec![ReviewRecord {
17613 attempts: 0,
17614 reviewer: 1,
17615 agent: "alpha".to_owned(),
17616 summary: String::new(),
17617 findings: Vec::new(),
17618 vote: None,
17619 failed: None,
17620 duration_ms: 0,
17621 }],
17622 e2e: Vec::new(),
17623 verify_retried: false,
17624 e2e_deferred: false,
17625 e2e_defer_reason: None,
17626 fix: None,
17627 blocking: 0,
17628 answered: 1,
17629 expected: 1,
17630 clean: true,
17631 progressed: false,
17632 vote_split: false,
17633 reconsideration: Vec::new(),
17634 verdict: None,
17635 };
17636 let state = state_with_round(round);
17637 let body = pr_message(&state, 'A').body;
17638 assert!(!body.contains("Open review findings"), "{body}");
17639 assert!(!body.contains("Declined"), "{body}");
17640 }
17641
17642 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17643 let mut state = RunState::new(
17644 PathBuf::from("/repo"),
17645 "main".to_owned(),
17646 "abc1234".to_owned(),
17647 instruction.to_owned(),
17648 Config::default(),
17649 );
17650 state.candidates.push(Candidate {
17651 index: 0,
17652 label: 'A',
17653 agent: "alpha".to_owned(),
17654 branch: "magi/x/A".to_owned(),
17655 worktree: PathBuf::from("/wt"),
17656 summary: summary.to_owned(),
17657 stat: String::new(),
17658 files: 1,
17659 commits: 1,
17660 empty: false,
17661 failed: None,
17662 verified_noop: None,
17663 folded: false,
17664 duration_ms: 0,
17665 });
17666 state
17667 }
17668
17669 fn review_state(subjects: &[&str]) -> RunState {
17670 let mut state = state_with_summary(
17671 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17672 "",
17673 );
17674 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17675 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17676 state
17677 }
17678
17679 #[tokio::test]
17682 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17683 ask_test_home();
17684 let tmp = tempfile::tempdir().unwrap();
17685 let repo = tmp.path().join("repo");
17686 std::fs::create_dir_all(&repo).unwrap();
17687 init_repo(&repo);
17688 let origin = tmp.path().join("origin.git");
17689 let g = |dir: &Path, args: &[&str]| {
17690 let out = std::process::Command::new("git")
17691 .args(args)
17692 .current_dir(dir)
17693 .quiet()
17694 .output()
17695 .expect("spawn git");
17696 assert!(
17697 out.status.success(),
17698 "git {args:?}: {}",
17699 String::from_utf8_lossy(&out.stderr)
17700 );
17701 };
17702 g(
17703 tmp.path(),
17704 &[
17705 "clone",
17706 "--bare",
17707 "-q",
17708 repo.to_str().unwrap(),
17709 origin.to_str().unwrap(),
17710 ],
17711 );
17712 g(
17713 &repo,
17714 &["remote", "add", "origin", origin.to_str().unwrap()],
17715 );
17716 let c1 = git::rev_parse(&repo, "main").await.unwrap();
17717
17718 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17720 g(&repo, &["add", "-A"]);
17721 g(
17722 &repo,
17723 &["commit", "-q", "-m", "chore(deps): update a crate"],
17724 );
17725 g(&repo, &["push", "-q", "origin", "main"]);
17726 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17727 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17728 g(&repo, &["add", "-A"]);
17729 g(
17730 &repo,
17731 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17732 );
17733 g(&repo, &["checkout", "-q", "main"]);
17734
17735 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17736 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17737 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17739 assert_eq!(fallback, c1);
17740
17741 let mut state = review_state(&[
17742 "chore(deps): update a crate",
17743 "fix(daemon): apply a chosen action",
17744 ]);
17745 state.repo = repo.clone();
17746 state.base_branch = "main".to_owned();
17747 state.base_commit = c1;
17748 refresh_reviewed_commits(&mut state, "feat/own").await;
17749 assert_eq!(
17750 state.reviewed_commits,
17751 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17752 );
17753 assert_eq!(
17754 review_title(&state).as_deref(),
17755 Some("fix(daemon): apply a chosen action")
17756 );
17757 assert_eq!(
17758 leaked_subjects(&state, "feat/own").await,
17759 Some(vec!["chore(deps): update a crate".to_owned()])
17760 );
17761 state.config.merge.remote = "nowhere".to_owned();
17764 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17765 }
17766
17767 struct SummaryFixture {
17771 _tmp: tempfile::TempDir,
17772 repo: PathBuf,
17773 reply: PathBuf,
17774 runner: Runner,
17775 }
17776
17777 fn summary_fixture() -> SummaryFixture {
17778 ask_test_home();
17779 let tmp = tempfile::tempdir().unwrap();
17780 let repo = tmp.path().join("repo");
17781 std::fs::create_dir_all(&repo).unwrap();
17782 init_repo(&repo);
17783 let origin = tmp.path().join("origin.git");
17784 let g = |args: &[&str]| {
17785 let out = std::process::Command::new("git")
17786 .args(args)
17787 .current_dir(&repo)
17788 .quiet()
17789 .output()
17790 .expect("spawn git");
17791 assert!(
17792 out.status.success(),
17793 "git {args:?}: {}",
17794 String::from_utf8_lossy(&out.stderr)
17795 );
17796 };
17797 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17798 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17799 g(&["push", "-q", "origin", "main"]);
17800 g(&["checkout", "-q", "-b", "feat/own"]);
17801 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17802 g(&["add", "-A"]);
17803 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17804 let reply = tmp.path().join("reply.txt");
17805 std::fs::write(
17806 &reply,
17807 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17808 )
17809 .unwrap();
17810 let mut runner = runner_at(RunStatus::Gating);
17811 let mut state = review_state(&["feat(own): add the own file"]);
17812 state.repo = repo.clone();
17813 state.base_branch = "main".to_owned();
17814 state.candidates[0].branch = "feat/own".to_owned();
17815 state.candidates[0].worktree = repo.clone();
17816 let mut author = spec("alpha");
17817 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17818 author
17819 .env
17820 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17821 state.config.agents = vec![author];
17822 runner.state = state;
17823 SummaryFixture {
17824 _tmp: tmp,
17825 repo,
17826 reply,
17827 runner,
17828 }
17829 }
17830
17831 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17832 let winner = f.runner.state.candidates[0].clone();
17833 f.runner
17834 .guarded_pr_message(&winner, None, true)
17835 .await
17836 .unwrap()
17837 .expect("an English description never parks")
17838 }
17839
17840 fn clean_round_with_minor_findings() -> ReviewRound {
17841 let mut round = passing_round(2, None);
17842 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17843 round.reviews = vec![ReviewRecord {
17844 attempts: 0,
17845 reviewer: 1,
17846 agent: "alpha".into(),
17847 summary: "内部レビュー".into(),
17848 findings: vec![
17849 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17850 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17851 ],
17852 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17853 failed: None,
17854 duration_ms: 0,
17855 }];
17856 round
17857 }
17858
17859 #[tokio::test]
17860 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17861 for review_only in [false, true] {
17862 let mut f = summary_fixture();
17863 if !review_only {
17864 f.runner.state.candidates[0].agent = "alpha".into();
17865 }
17866 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17867 assert!(f.runner.state.reviews[0].clean);
17868 assert!(f.runner.state.reviews[0].fix.is_none());
17869 assert!(f.runner.state.open_findings().is_empty());
17870 assert_eq!(
17871 crate::pr_outcome::required_finding_ids(&f.runner.state),
17872 ["R2-1-1", "R2-1-2"]
17873 );
17874 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17875 let evidence = crate::pr_outcome::summary_evidence(
17876 &f.runner.state,
17877 crate::pr_outcome::FinalHead::At(&head),
17878 );
17879 assert!(
17880 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17881 );
17882 let omitted = posted(&mut f).await;
17884 assert!(omitted.body.contains("No description"));
17885 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17886 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17887 assert!(f.runner.state.change_summary.is_none());
17888 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17889 let described = posted(&mut f).await;
17890 assert!(described.body.contains("improve documentation for callers"));
17891 assert!(described.body.contains("remaining edge case"));
17892 assert!(described.body.contains("## Open review findings"));
17893 assert!(!described.body.contains("未修正"));
17894 }
17895 }
17896
17897 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17898 let mut f = summary_fixture();
17899 if !review_only {
17900 f.runner.state.candidates[0].agent = "alpha".into();
17901 }
17902 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17903 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17904 fixed
17905 .fix
17906 .as_mut()
17907 .unwrap()
17908 .rejected
17909 .push(crate::verdict::Rejection {
17910 id: "R1-1-1".into(),
17911 why: "呼び出されないコードのため辞退".into(),
17912 });
17913 let mut clean = clean_round_with_minor_findings();
17914 clean.verified_head = Some(head.clone());
17915 f.runner.state.reviews = vec![fixed, clean];
17916 ok_gate(&mut f.runner.state, Some(&head));
17917 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17918 f.runner.state.config.agents.push(scripted("beta", reply));
17919 f.runner.state.config.roles.language_judge =
17920 Some(crate::config::AgentChoice::One("beta".into()));
17921 f
17922 }
17923
17924 #[tokio::test]
17925 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17926 for review_only in [false, true] {
17927 let mut f = rewrite_fixture_with_findings(review_only,
17928 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17929 let m = posted(&mut f).await;
17930 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17931 let expected =
17932 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17933 assert!(m.body.contains(&expected));
17934 assert!(m.body.contains("## Review rounds"));
17935 assert!(
17936 m.body
17937 .contains("Gate (`verify.gate`): passed on the final commit")
17938 );
17939 assert!(
17940 m.body
17941 .contains("Tests (`verify.e2e`): passed on the final commit")
17942 );
17943 let c = f.runner.state.change_summary.as_ref().unwrap();
17944 assert!(
17945 m.body
17946 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17947 );
17948 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17949 assert!(crate::pr_outcome::covers_required_findings(
17950 &f.runner.state,
17951 &m.body
17952 ));
17953 assert!(m.body.contains("documentation remains incomplete"));
17954 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17955 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17956 assert!(f.runner.state.github_text.is_none());
17957 }
17958 }
17959
17960 #[tokio::test]
17961 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17962 for review_only in [false, true] {
17963 for reply in [
17964 "TITLE: fix: public description\nA short English description.",
17965 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17966 ] {
17967 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17968 let winner = f.runner.state.candidates[0].clone();
17969 let posted = f
17970 .runner
17971 .guarded_pr_message(&winner, None, true)
17972 .await
17973 .unwrap();
17974 assert!(posted.is_none());
17975 assert!(f.runner.state.parked);
17976 let withheld = f.runner.state.github_text.as_ref().unwrap();
17977 assert!(
17978 withheld.rewrite.is_none(),
17979 "an incomplete rewrite must not be selectable"
17980 );
17981 let q = ask::Questions::open()
17982 .get(withheld.question.as_ref().unwrap())
17983 .unwrap();
17984 assert!(
17985 !q.choices
17986 .contains(&crate::github_text::USE_REWRITE.to_owned())
17987 );
17988 assert!(
17989 q.detail.contains("R1-1-1")
17990 && q.detail.contains("R2-1-1")
17991 && q.detail.contains("R2-1-2")
17992 );
17993 }
17994 }
17995 }
17996
17997 #[tokio::test]
17998 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17999 for review_only in [false, true] {
18000 let mut f = summary_fixture();
18001 if !review_only {
18002 f.runner.state.candidates[0].agent = "alpha".into();
18003 }
18004 f.runner.state.instruction = "内部タスクの説明".into();
18005 f.runner.state.candidates[0].summary = "内部の候補報告".into();
18006 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18007 f.runner.state.reviews = vec![
18008 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18009 round_with_notes(2, Some("二回目の内部修正報告です。")),
18010 passing_round(3, Some(&head)),
18011 ];
18012 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18013 ok_gate(&mut f.runner.state, Some(&head));
18014 let captured = f.repo.join("public-prompt.txt");
18015 let author = &mut f.runner.state.config.agents[0];
18016 author.command = vec![
18017 "sh".into(),
18018 "-c".into(),
18019 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18020 ];
18021 author.env.insert(
18022 "PROMPT_FILE".into(),
18023 captured.to_string_lossy().into_owned(),
18024 );
18025 let m = posted(&mut f).await;
18026 assert!(m.body.contains("Adds `own.txt`"));
18027 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18028 assert!(m.body.contains("R1-1-1"));
18029 assert!(
18030 m.body
18031 .contains("Gate (`verify.gate`): passed on the final commit")
18032 );
18033 assert!(
18034 m.body
18035 .contains("Tests (`verify.e2e`): passed on the final commit")
18036 );
18037 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18038 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18039 let prompt = std::fs::read_to_string(captured).unwrap();
18040 assert!(prompt.contains(&format!(
18041 "git diff {} {head}",
18042 f.runner.state.change_summary.as_ref().unwrap().base
18043 )));
18044 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18045 assert_eq!(
18046 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18047 "二回目の内部修正報告です。"
18048 );
18049 }
18050 }
18051
18052 #[tokio::test]
18053 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18054 for review_only in [false, true] {
18055 let mut f = summary_fixture();
18056 if !review_only {
18057 f.runner.state.candidates[0].agent = "alpha".into();
18058 }
18059 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18060 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18061 std::fs::write(&f.reply, "no usable description").unwrap();
18062 let failed = posted(&mut f).await;
18063 assert!(
18064 failed
18065 .body
18066 .contains("No description of what this change does could be generated")
18067 );
18068 assert!(failed.body.is_ascii());
18069 assert!(f.runner.state.github_text.is_none());
18070 assert!(f.runner.state.change_summary.is_none());
18071 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18072 let evidence = crate::pr_outcome::summary_evidence(
18073 &f.runner.state,
18074 crate::pr_outcome::FinalHead::At(&head),
18075 );
18076 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18077 assert!(
18078 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
18079 .exists()
18080 );
18081 f.runner.state =
18082 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
18083 std::fs::write(
18084 &f.reply,
18085 "TITLE: feat(own): add the own file\nAdds the committed own file.",
18086 )
18087 .unwrap();
18088 let retried = posted(&mut f).await;
18089 assert!(retried.body.contains("Adds the committed own file."));
18090 assert_eq!(
18091 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
18092 "日本語の修正報告"
18093 );
18094 }
18095 }
18096
18097 #[tokio::test]
18098 async fn same_head_verification_and_findings_invalidate_public_prose() {
18099 let mut f = summary_fixture();
18100 posted(&mut f).await;
18101 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18102 let original = f
18103 .runner
18104 .state
18105 .change_summary
18106 .as_ref()
18107 .unwrap()
18108 .evidence
18109 .clone();
18110 ok_gate(&mut f.runner.state, Some(&head));
18111 std::fs::write(
18112 &f.reply,
18113 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
18114 )
18115 .unwrap();
18116 let verified = posted(&mut f).await;
18117 assert!(verified.body.contains("now verified"));
18118 assert_ne!(
18119 f.runner.state.change_summary.as_ref().unwrap().evidence,
18120 original
18121 );
18122 let mut round = round_with_notes(1, Some("内部報告"));
18123 round.reviews.push(ReviewRecord {
18124 attempts: 0,
18125 reviewer: 1,
18126 agent: "alpha".into(),
18127 summary: "内部レビュー".into(),
18128 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
18129 vote: None,
18130 failed: None,
18131 duration_ms: 0,
18132 });
18133 round
18134 .fix
18135 .as_mut()
18136 .unwrap()
18137 .rejected
18138 .push(crate::verdict::Rejection {
18139 id: "R1-1-1".into(),
18140 why: "呼び出し元から到達できない".into(),
18141 });
18142 f.runner.state.reviews = vec![round];
18143 let omitted = posted(&mut f).await;
18145 assert!(omitted.body.contains("No description"));
18146 assert!(omitted.body.contains("R1-1-1"));
18147 assert!(!omitted.body.contains("到達"));
18148 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
18149 let described = posted(&mut f).await;
18150 assert!(described.body.contains("unreachable from callers"));
18151 assert!(described.body.contains("R1-1-1"));
18152 assert!(!described.body.contains("到達"));
18153 let evidence = f
18154 .runner
18155 .state
18156 .change_summary
18157 .as_ref()
18158 .unwrap()
18159 .evidence
18160 .clone();
18161 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
18162 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
18163 assert!(posted(&mut f).await.body.contains("rationale updated"));
18164 assert_ne!(
18165 f.runner.state.change_summary.as_ref().unwrap().evidence,
18166 evidence
18167 );
18168 f.runner
18170 .state
18171 .change_summary
18172 .as_mut()
18173 .unwrap()
18174 .evidence
18175 .clear();
18176 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
18177 assert!(posted(&mut f).await.body.contains("current specification"));
18178 }
18179
18180 #[tokio::test]
18181 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
18182 let mut f = summary_fixture();
18183 let m = posted(&mut f).await;
18184 assert_eq!(m.title, "feat(own): add the own file");
18185 assert!(
18186 m.body
18187 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
18188 "{}",
18189 m.body
18190 );
18191 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
18192 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
18193 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18194 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18195 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
18196 let kept = f.runner.state.change_summary.as_ref().unwrap();
18197 assert_eq!(
18198 (kept.head.as_str(), kept.base.as_str()),
18199 (head.as_str(), base.as_str())
18200 );
18201 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
18202 }
18203
18204 #[tokio::test]
18205 async fn review_only_summary_receives_the_configured_github_language() {
18206 let mut f = summary_fixture();
18207 f.runner.state.config.graph.github_language = "ja".into();
18208 let captured = f.repo.join("summary-prompt.txt");
18209 let author = &mut f.runner.state.config.agents[0];
18210 author.command = vec![
18211 "sh".into(),
18212 "-c".into(),
18213 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18214 ];
18215 author.env.insert(
18216 "PROMPT_FILE".into(),
18217 captured.to_string_lossy().into_owned(),
18218 );
18219 std::fs::write(
18220 &f.reply,
18221 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18222 )
18223 .unwrap();
18224 let m = posted(&mut f).await;
18225 let prompt = std::fs::read_to_string(captured).unwrap();
18226 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18227 assert!(!prompt.contains("in English"));
18228 assert_eq!(m.title, "feat(own): ファイルを追加");
18229 assert!(m.body.contains("own.txt を追加します。"));
18230 }
18231
18232 #[tokio::test]
18233 async fn review_fixes_come_after_the_whole_change_summary() {
18234 let mut f = summary_fixture();
18235 f.runner.state.reviews = vec![
18236 round_with_notes(1, Some("handled the PATH case")),
18237 round_with_notes(2, None),
18238 ];
18239 let m = posted(&mut f).await;
18240 let summary = m.body.find("## Summary").expect("summary");
18241 let fixes = m.body.find("## Review rounds").expect("review rounds");
18242 assert!(summary < fixes, "{}", m.body);
18243 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18244 assert!(m.body.contains("## Final verification"), "{}", m.body);
18245 }
18246
18247 #[tokio::test]
18248 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18249 let mut f = summary_fixture();
18250 let first = posted(&mut f).await;
18251 let first_head = f.runner.state.change_summary.clone().unwrap().head;
18252 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18254 let again = posted(&mut f).await;
18255 assert_eq!((again.title, again.body), (first.title, first.body));
18256
18257 let g = |args: &[&str]| {
18258 let out = std::process::Command::new("git")
18259 .args(args)
18260 .current_dir(&f.repo)
18261 .quiet()
18262 .output()
18263 .expect("spawn git");
18264 assert!(out.status.success(), "git {args:?}");
18265 };
18266 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18267 g(&["add", "-A"]);
18268 g(&["commit", "-q", "-m", "fix(own): handle more"]);
18269 let second = posted(&mut f).await;
18270 let kept = f.runner.state.change_summary.clone().unwrap();
18271 assert_ne!(kept.head, first_head);
18272 assert!(second.body.contains("Not read yet."), "{}", second.body);
18273 assert!(!second.body.contains("so the first summary is read"));
18274 assert_eq!(second.title, "feat(own): other");
18275 }
18276
18277 #[tokio::test]
18278 async fn an_unavailable_summary_is_stated_and_never_invented() {
18279 let mut f = summary_fixture();
18280 std::fs::write(&f.reply, "no title line here").unwrap();
18281 let m = posted(&mut f).await;
18282 assert!(
18283 m.body
18284 .contains("No description of what this change does could be generated"),
18285 "{}",
18286 m.body
18287 );
18288 assert!(m.body.contains("internal reports remain in local records"));
18289 assert!(f.runner.state.change_summary.is_none());
18290 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18291 let winner = f.runner.state.candidates[0].clone();
18293 let preview = f
18294 .runner
18295 .guarded_pr_message(&winner, None, false)
18296 .await
18297 .unwrap()
18298 .unwrap();
18299 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18300 }
18301
18302 #[test]
18303 fn pr_message_review_single_commit_uses_its_subject() {
18304 let state = review_state(&["feat(nats): per-role user"]);
18305 let m = pr_message(&state, 'A');
18306 assert_eq!(m.title, "feat(nats): per-role user");
18307 assert!(!m.body.contains("Review the work already"), "{}", m.body);
18308 assert!(m.body.contains("## Commits under review"), "{}", m.body);
18309 }
18310
18311 #[test]
18312 fn pr_message_review_multi_commit_takes_the_oldest() {
18313 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18314 let m = pr_message(&state, 'A');
18315 assert_eq!(m.title, "feat: the change");
18316 for s in ["feat: the change", "fix: typo", "fix: again"] {
18317 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18318 }
18319 }
18320
18321 #[test]
18322 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18323 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18324 let mut state = review_state(&[first, "fix: later fixup"]);
18325 state.candidates[0].branch = "機能/ログイン".to_owned();
18326 let m = pr_message(&state, 'A');
18327 assert!(
18328 m.title.starts_with("chore: land candidate A of run"),
18329 "{}",
18330 m.title
18331 );
18332 }
18333 }
18334
18335 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18336 BranchFacts {
18337 commits: commits
18338 .iter()
18339 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18340 .collect(),
18341 stat: stat.to_owned(),
18342 }
18343 }
18344
18345 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18346 let mut r = review_round(true, 0, 1, 1, true, true);
18347 r.round = round;
18348 r.fix = notes.map(|n| FixRecord {
18349 agent: "fixer".to_owned(),
18350 addressed: Vec::new(),
18351 rejected: Vec::new(),
18352 notes: n.to_owned(),
18353 committed: true,
18354 failed: None,
18355 duration_ms: 0,
18356 continuation: None,
18357 });
18358 r
18359 }
18360
18361 #[test]
18362 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18363 let state = review_state(&["ignored"]);
18364 let f = facts(
18365 &[
18366 (
18367 "fix(login): resolve PATH on macOS",
18368 "Login shells skip rc files.",
18369 ),
18370 ("fix: address review", ""),
18371 ],
18372 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18373 );
18374 let m = pr_message_with(&state, 'A', Some(&f));
18375 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18376 assert!(
18377 m.body
18378 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
18379 "{}",
18380 m.body
18381 );
18382 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18383 assert!(m.body.contains("## Diff stat"), "{}", m.body);
18384 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18385 for banned in [
18386 "Review the work already",
18387 "no task statement",
18388 "Original task",
18389 ] {
18390 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18391 }
18392 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18393 }
18394
18395 #[test]
18396 fn pr_message_review_truncates_a_huge_first_commit_body() {
18397 let state = review_state(&["ignored"]);
18398 let f = facts(
18399 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18400 "s",
18401 );
18402 let m = pr_message_with(&state, 'A', Some(&f));
18403 assert!(m.body.len() < 30_000, "{}", m.body.len());
18404 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18405 assert!(
18406 m.body.contains("- ... 1 more commit(s)"),
18407 "{}",
18408 m.body.len()
18409 );
18410 assert!(m.body.ends_with("magi:candidate-a\n"));
18411 }
18412
18413 #[test]
18414 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18415 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18416 assert_eq!(m.title, "feat: x");
18417 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18418 assert!(!m.body.contains("Diff stat"), "{}", m.body);
18419 assert!(!m.body.contains("no task statement"), "{}", m.body);
18420 }
18421
18422 #[test]
18423 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18424 let mut state = review_state(&["日本語の件名"]);
18425 state.candidates[0].branch = "fix/macos-login-path".to_owned();
18426 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18427 state.candidates[0].branch = "機能/ログイン".to_owned();
18428 assert!(
18429 pr_message(&state, 'A')
18430 .title
18431 .starts_with("chore: land candidate A")
18432 );
18433 }
18434
18435 #[test]
18436 fn pr_message_review_fixes_survive_a_clean_final_round() {
18437 let mut state = review_state(&["feat: x"]);
18438 state.reviews = vec![
18439 round_with_notes(1, Some("handled the PATH case")),
18440 round_with_notes(2, None),
18441 ];
18442 let body = pr_message(&state, 'A').body;
18443 assert!(
18444 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18445 "{body}"
18446 );
18447 assert!(body.contains("### Round 1"), "{body}");
18448
18449 state.reviews = vec![
18450 round_with_notes(1, Some("first fix")),
18451 round_with_notes(2, Some("")),
18452 round_with_notes(3, Some("second fix")),
18453 round_with_notes(4, None),
18454 ];
18455 let body = pr_message(&state, 'A').body;
18456 assert!(
18457 body.contains("### Round 1") && !body.contains("first fix"),
18458 "{body}"
18459 );
18460 assert!(
18461 body.contains("### Round 3") && !body.contains("second fix"),
18462 "{body}"
18463 );
18464 assert!(body.contains("### Round 2"), "{body}");
18465 }
18466
18467 const FINAL: &str = "abcdef1234567890";
18468
18469 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18470 let mut r = round_with_notes(round, None);
18471 r.verified_head = verified.map(str::to_owned);
18472 r
18473 }
18474
18475 fn ok_gate(state: &mut RunState, head: Option<&str>) {
18476 state.gate = vec![CommandOutcome {
18477 command: "gate".to_owned(),
18478 code: Some(0),
18479 output_tail: String::new(),
18480 duration_ms: 0,
18481 resource_blocked: false,
18482 }];
18483 state.gate_ran = true;
18484 state.gate_head = head.map(str::to_owned);
18485 }
18486
18487 fn body_at(state: &RunState) -> String {
18488 pr_message_raw(
18489 state,
18490 'A',
18491 None,
18492 crate::pr_outcome::FinalHead::At(FINAL),
18493 ChangeNote::NotAsked,
18494 )
18495 .body
18496 }
18497
18498 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18499
18500 #[test]
18501 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18502 let mut state = review_state(&["feat: x"]);
18503 state.reviews = vec![
18504 round_with_notes(1, Some(SANDBOXED)),
18505 passing_round(2, Some(FINAL)),
18506 ];
18507 ok_gate(&mut state, Some(FINAL));
18508 let body = body_at(&state);
18509 assert!(!body.contains("Handled the PATH case."), "{body}");
18510 assert!(!body.contains("could not commit"), "{body}");
18511 assert!(!body.contains("could not run"), "{body}");
18512 assert!(
18513 body.contains("The fix is committed on the branch"),
18514 "{body}"
18515 );
18516 assert!(!body.contains("sandbox"), "{body}");
18517 assert!(
18518 body.contains("Tests (`verify.e2e`): passed on the final commit"),
18519 "{body}"
18520 );
18521 assert!(
18522 body.contains("Gate (`verify.gate`): passed on the final commit"),
18523 "{body}"
18524 );
18525 assert!(
18527 state.reviews[0]
18528 .fix
18529 .as_ref()
18530 .unwrap()
18531 .notes
18532 .contains("could not commit")
18533 );
18534 }
18535
18536 #[test]
18537 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18538 let mut state = review_state(&["feat: x"]);
18539 let mut r1 = round_with_notes(1, Some(SANDBOXED));
18540 r1.fix.as_mut().unwrap().committed = false;
18541 let mut r2 = passing_round(2, Some(FINAL));
18542 r2.e2e[0].code = Some(1);
18543 state.reviews = vec![r1, r2];
18544 let body = body_at(&state);
18545 assert!(!body.contains("could not commit"), "{body}");
18546 assert!(!body.contains("could not run"), "{body}");
18547 assert!(body.contains("This round produced no commit"), "{body}");
18548 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18549 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18550 assert!(!body.contains("passed"), "{body}");
18551 }
18552
18553 #[test]
18554 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18555 let mut state = review_state(&["feat: x"]);
18556 state.reviews = vec![
18557 round_with_notes(1, Some(SANDBOXED)),
18558 passing_round(2, Some("0123456789abcdef")),
18559 ];
18560 ok_gate(&mut state, Some("0123456789abcdef"));
18561 let body = body_at(&state);
18562 assert!(!body.contains("HTTP tests could not run"), "{body}");
18563 assert!(!body.contains("could not commit"), "{body}");
18565 assert!(
18566 body.contains("passed on an earlier commit (`0123456`)"),
18567 "{body}"
18568 );
18569 assert!(!body.contains("passed on the final commit"), "{body}");
18570
18571 ok_gate(&mut state, None);
18573 let body = body_at(&state);
18574 assert!(
18575 body.contains("passed on a commit that was not recorded"),
18576 "{body}"
18577 );
18578 }
18579
18580 #[test]
18581 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18582 let state = state_with_summary(
18583 "add retries to the client",
18584 "TITLE: feat: retries\n\nDid it.",
18585 );
18586 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18587 assert_eq!(m.title, "feat: retries");
18588 assert!(
18589 m.body.contains("<summary>Original task</summary>"),
18590 "{}",
18591 m.body
18592 );
18593 assert!(!m.body.contains("Commits under review"), "{}", m.body);
18594 assert!(
18595 m.body
18596 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18597 "{}",
18598 m.body
18599 );
18600 }
18601
18602 #[test]
18603 fn pr_message_review_bounds_a_long_english_subject() {
18604 let long = format!("feat: {}", "word ".repeat(100));
18605 let m = pr_message(&review_state(&[&long]), 'A');
18606 assert!(m.title.starts_with("feat: word"), "{}", m.title);
18607 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18608 }
18609
18610 #[test]
18611 fn pr_message_implementation_run_is_unchanged_by_review_support() {
18612 let state = state_with_summary("add retries\n\ndetails", "- did some things");
18613 let m = pr_message(&state, 'A');
18614 assert_eq!(m.title, "add retries");
18615 assert!(m.body.contains("<summary>Original task</summary>"));
18616 assert!(!m.body.contains("Commits under review"));
18617 assert_eq!(landing_subject_source(&state), state.instruction);
18618 }
18619
18620 #[test]
18621 fn review_run_squash_subject_is_the_change_not_the_prompt() {
18622 let state = review_state(&["feat: the change", "fix: typo"]);
18623 let source = landing_subject_source(&state);
18624 assert_eq!(land::merge_subject("", &source), "feat: the change");
18625 assert_eq!(
18626 land::merge_subject("magi: candidate A (uncommitted work)", &source),
18627 "feat: the change"
18628 );
18629 assert_eq!(
18631 land::merge_subject("feat: renamed by hand", &source),
18632 "feat: renamed by hand"
18633 );
18634 let blank = review_state(&["日本語"]);
18635 assert!(
18636 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18637 );
18638 }
18639
18640 #[test]
18641 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18642 let state = review_state(&["feat: the change"]);
18643 let source = landing_subject_source(&state);
18644 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18645 assert_eq!(
18646 land::merge_subject(landing_title(&state, old), &source),
18647 "feat: the change"
18648 );
18649 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18650 let task = state_with_summary("add retries", "");
18651 assert_eq!(landing_title(&task, old), old);
18652 }
18653
18654 #[test]
18655 fn pr_message_describes_the_change_not_the_task() {
18656 let state = state_with_summary(
18657 "今回やってほしいこと: results projector を直す",
18658 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18659 );
18660 let m = pr_message(&state, 'A');
18661 assert_eq!(m.title, "fix(web): batch the runs list reads");
18662 assert!(
18663 m.body.starts_with("## Summary\n\n- reads run.json once"),
18664 "{}",
18665 m.body
18666 );
18667 assert!(!m.body.contains("TITLE:"), "{}", m.body);
18668 let task_at = m.body.find("今回やってほしいこと").unwrap();
18669 let details_at = m.body.find("<details>").unwrap();
18670 assert!(
18671 details_at < task_at,
18672 "the task lives inside <details>: {}",
18673 m.body
18674 );
18675 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18676 assert!(m.body.contains("magi:candidate-a"));
18677 }
18678
18679 #[test]
18680 fn pr_message_falls_back_to_the_task_without_a_title_line() {
18681 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18682 let m = pr_message(&state, 'A');
18683 assert_eq!(m.title, "add retries");
18684 assert!(
18685 m.body.contains("## Summary\n\n- did some things"),
18686 "{}",
18687 m.body
18688 );
18689
18690 let none = RunState::new(
18691 PathBuf::from("/repo"),
18692 "main".to_owned(),
18693 "abc1234".to_owned(),
18694 "add retries".to_owned(),
18695 Config::default(),
18696 );
18697 let m = pr_message(&none, 'A');
18698 assert_eq!(m.title, "add retries");
18699 assert!(!m.body.contains("## Summary"), "{}", m.body);
18700 }
18701
18702 #[test]
18703 fn pr_message_refuses_the_candidate_commit_subject() {
18704 for bad in [
18705 "TITLE: magi: candidate A (uncommitted work)",
18706 "TITLE: chore: stuff (uncommitted work)",
18707 "TITLE: ",
18708 ] {
18709 let state = state_with_summary("add retries", bad);
18710 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18711 }
18712 }
18713
18714 #[test]
18715 fn pr_message_bounds_a_very_long_task_and_title() {
18716 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18717 let state = state_with_summary(&long, "- nothing");
18718 let m = pr_message(&state, 'A');
18719 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18720 assert!(!m.title.contains('\n'));
18721
18722 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18723 let m = pr_message(&state, 'A');
18724 assert!(m.title.starts_with("feat: "));
18725 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18726 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18727 }
18728
18729 fn long_title_of(instruction: &str) -> String {
18730 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18731 }
18732
18733 #[test]
18734 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18735 let first = "Make the landing path keep a readable title for long tasks";
18736 let line = format!(
18737 "{first}. {}",
18738 "And then keep going with more words ".repeat(20)
18739 );
18740 let t = long_title_of(&line);
18741 assert_eq!(t, first);
18742 assert!(!t.starts_with("chore: land"));
18743 }
18744
18745 #[test]
18746 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18747 let line = "word ".repeat(200);
18748 let t = long_title_of(&line);
18749 assert!(t.ends_with("word..."), "{t}");
18750 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18751 }
18752
18753 #[test]
18754 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18755 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18756 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18757 }
18758 }
18759
18760 #[test]
18761 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18762 let mut state = state_with_summary("再試行を追加する", "- nothing");
18763 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18764 state.config.graph.github_language = "ja".into();
18765 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18766 }
18767
18768 #[test]
18769 fn pr_message_title_limit_is_exact() {
18770 let at = "a".repeat(PR_TITLE_MAX);
18771 assert_eq!(long_title_of(&at), at);
18772 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18773 assert!(over.ends_with("..."), "{over}");
18774 assert_eq!(over.chars().count(), PR_TITLE_MAX);
18775 }
18776
18777 #[test]
18778 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18779 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18780 let t = long_title_of(&line);
18781 assert!(t.ends_with("..."), "{t}");
18782 assert!(t.is_ascii(), "{t}");
18783 }
18784
18785 #[test]
18786 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18787 let line = format!(
18788 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18789 "plus more filler words ".repeat(20)
18790 );
18791 let t = long_title_of(&line);
18792 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18793 }
18794
18795 #[test]
18796 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18797 let line = format!("\n\n# {}", "title words ".repeat(40));
18798 let state = state_with_summary(&line, "- nothing");
18799 let m = pr_message(&state, 'A');
18800 assert!(m.title.starts_with("title words"), "{}", m.title);
18801 assert_eq!(
18802 land::merge_subject(&m.title, &landing_subject_source(&state)),
18803 m.title
18804 );
18805 assert_eq!(
18807 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18808 "feat: renamed by hand"
18809 );
18810 }
18811
18812 #[tokio::test]
18813 async fn github_text_rewrite_is_bounded_and_falls_back() {
18814 crate::run::pin_test_home();
18815 for (reply, accepted) in [
18816 (
18817 "TITLE: fix: retries\nAdd retries for failed requests.",
18818 true,
18819 ),
18820 (
18821 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18822 false,
18823 ),
18824 (
18825 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18826 true,
18827 ),
18828 (
18829 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18830 true,
18831 ),
18832 (
18833 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18834 true,
18835 ),
18836 (
18837 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18838 false,
18839 ),
18840 ("TITLE: fix: retries\n", false),
18841 ] {
18842 let dir = tempfile::tempdir().unwrap();
18843 let mut runner = runner_at(RunStatus::Gating);
18844 runner.state = state_with_summary(
18845 "add retries",
18846 "TITLE: fix: retries\n日本語の説明を書きます。",
18847 );
18848 runner.state.repo = dir.path().to_owned();
18849 runner.state.candidates[0].worktree = dir.path().to_owned();
18850 let mut author = spec("alpha");
18851 author.command = vec![
18852 "sh".into(),
18853 "-c".into(),
18854 "printf '%s' \"$REWRITE_REPLY\"".into(),
18855 ];
18856 author.env.insert("REWRITE_REPLY".into(), reply.into());
18857 runner.state.config.agents = vec![author];
18858 seed_public_summary(&mut runner);
18859 let winner = runner.state.candidates[0].clone();
18860 let first = runner
18861 .guarded_pr_message(&winner, None, true)
18862 .await
18863 .unwrap();
18864 let message = if accepted {
18865 first.expect("a passing rewrite never asks")
18866 } else {
18867 assert!(first.is_none());
18869 assert!(runner.state.parked);
18870 let store = ask::Questions::open();
18871 let asked: Vec<_> = store
18872 .list()
18873 .into_iter()
18874 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18875 .collect();
18876 assert_eq!(asked.len(), 1);
18877 assert!(!asked[0].detail.contains("secret"));
18878 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18879 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18880 assert_eq!(
18881 asked[0]
18882 .choices
18883 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18884 rewrite.is_some()
18885 );
18886 store
18889 .update(&asked[0].id, |q| {
18890 q.abandon("test");
18891 Ok(())
18892 })
18893 .unwrap();
18894 let again = runner
18895 .guarded_pr_message(&winner, None, true)
18896 .await
18897 .unwrap()
18898 .expect("an abandoned question falls back");
18899 let n = store
18900 .list()
18901 .into_iter()
18902 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18903 .count();
18904 assert_eq!(n, 1, "asked once per rejected text");
18905 again
18906 };
18907 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18908 assert!(!message.body.contains("secret"));
18909 assert!(!message.body.contains("prefix"));
18910 assert!(!message.title.contains("secret"));
18911 assert_eq!(
18912 message.body.contains("Add retries for failed requests."),
18913 accepted
18914 );
18915 assert_eq!(
18916 runner.state.seats["github-text-rewrite"].turns, 1,
18917 "only one rewrite invocation"
18918 );
18919 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18920 if accepted {
18921 assert!(runner.state.github_text.is_none());
18922 assert!(!runner.state.parked);
18923 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18924 assert!(message.body.contains("## Final verification"));
18925 assert!(message.body.contains("Evidence: final diff"));
18926 assert!(
18927 message
18928 .body
18929 .contains(&format!("magi:run/{}", runner.state.id))
18930 );
18931 }
18932 if !accepted {
18933 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18934 assert!(
18935 message
18936 .body
18937 .contains(&format!("magi:run/{}", runner.state.id))
18938 );
18939 }
18940 }
18941 }
18942
18943 fn scripted(id: &str, reply: &str) -> AgentSpec {
18944 let mut a = spec(id);
18945 a.command = vec![
18946 "sh".into(),
18947 "-c".into(),
18948 "printf '%s' \"$REWRITE_REPLY\"".into(),
18949 ];
18950 a.env.insert("REWRITE_REPLY".into(), reply.into());
18951 a
18952 }
18953
18954 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18955 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18956
18957 fn seed_public_summary(runner: &mut Runner) {
18959 let repo = &runner.state.repo;
18960 let g = |args: &[&str]| {
18961 let out = std::process::Command::new("git")
18962 .args(args)
18963 .current_dir(repo)
18964 .quiet()
18965 .output()
18966 .unwrap();
18967 assert!(out.status.success(), "git {args:?}");
18968 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18969 };
18970 g(&["init", "-q", "-b", "main"]);
18971 g(&[
18972 "-c",
18973 "user.name=Test",
18974 "-c",
18975 "user.email=test@example.invalid",
18976 "commit",
18977 "-q",
18978 "--allow-empty",
18979 "-m",
18980 "feat: fixture",
18981 ]);
18982 let head = g(&["rev-parse", "HEAD"]);
18983 g(&["update-ref", "refs/remotes/origin/main", &head]);
18984 let evidence = crate::pr_outcome::summary_evidence(
18985 &runner.state,
18986 crate::pr_outcome::FinalHead::At(&head),
18987 );
18988 let generated = &runner.state.candidates[0].summary;
18989 runner.state.change_summary = Some(crate::run::ChangeSummary {
18990 base: head.clone(),
18991 head,
18992 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18993 title: summary_title(generated).unwrap(),
18994 body: summary_without_title(generated),
18995 });
18996 }
18997
18998 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18999 let mut runner = runner_at(RunStatus::Gating);
19000 runner.state = state_with_summary("add retries", JA_SUMMARY);
19001 if review_only {
19002 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19003 }
19004 runner.state.repo = dir.to_owned();
19005 runner.state.candidates[0].worktree = dir.to_owned();
19006 runner.state.config.agents = agents;
19007 seed_public_summary(&mut runner);
19008 runner
19009 }
19010
19011 fn gate_questions(runner: &Runner) -> usize {
19012 ask::Questions::open()
19013 .list()
19014 .into_iter()
19015 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19016 .count()
19017 }
19018
19019 #[tokio::test]
19020 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19021 crate::run::pin_test_home();
19022 let dir = tempfile::tempdir().unwrap();
19023 let mut runner = rewrite_runner(
19024 dir.path(),
19025 true,
19026 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19027 );
19028 runner.state.config.roles.language_judge =
19029 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19030 let winner = runner.state.candidates[0].clone();
19031 let msg = runner
19032 .guarded_pr_message(&winner, None, true)
19033 .await
19034 .unwrap()
19035 .expect("an English rewrite posts without asking");
19036 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19037 assert!(msg.body.contains("<details>kept</details>"));
19038 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19039 assert_eq!(gate_questions(&runner), 0);
19040 assert!(runner.state.github_text.is_none());
19041 assert!(!runner.state.seats.contains_key("impl-A"));
19042 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19043 }
19044
19045 #[tokio::test]
19046 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19047 crate::run::pin_test_home();
19048 let dir = tempfile::tempdir().unwrap();
19049 let mut runner = rewrite_runner(
19050 dir.path(),
19051 false,
19052 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19053 );
19054 runner.state.config.roles.language_judge =
19055 Some(crate::config::AgentChoice::One("beta".into()));
19056 runner.state.seats.insert(
19057 "impl-A".into(),
19058 SeatState::new("impl-A", "alpha", runner.state.seed),
19059 );
19060 let winner = runner.state.candidates[0].clone();
19061 let msg = runner
19062 .guarded_pr_message(&winner, None, true)
19063 .await
19064 .unwrap()
19065 .expect("rewritten");
19066 assert!(msg.body.contains("<details>kept</details>"));
19067 assert_eq!(runner.state.seats["impl-A"].turns, 0);
19068 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19069 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19070 }
19071
19072 #[tokio::test]
19073 async fn rewrite_reuses_a_resumable_implementer_session() {
19074 crate::run::pin_test_home();
19075 let dir = tempfile::tempdir().unwrap();
19076 let mut runner = rewrite_runner(
19077 dir.path(),
19078 false,
19079 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
19080 );
19081 runner.state.config.roles.language_judge =
19082 Some(crate::config::AgentChoice::One("beta".into()));
19083 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19084 seat.turns = 1;
19085 runner.state.seats.insert("impl-A".into(), seat);
19086 let winner = runner.state.candidates[0].clone();
19087 runner
19088 .guarded_pr_message(&winner, None, true)
19089 .await
19090 .unwrap()
19091 .expect("rewritten");
19092 assert_eq!(runner.state.seats["impl-A"].turns, 2);
19093 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
19094 }
19095
19096 #[tokio::test]
19097 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
19098 crate::run::pin_test_home();
19101 let dir = tempfile::tempdir().unwrap();
19102 let mut exhausted = spec("alpha");
19103 exhausted.command = vec!["false".into()];
19104 let mut runner = rewrite_runner(
19105 dir.path(),
19106 false,
19107 vec![exhausted, scripted("beta", EN_REWRITE)],
19108 );
19109 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19111 "alpha".into(),
19112 "beta".into(),
19113 ]));
19114 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19115 seat.turns = 1;
19116 runner.state.seats.insert("impl-A".into(), seat);
19117 let winner = runner.state.candidates[0].clone();
19118 let msg = runner
19119 .guarded_pr_message(&winner, None, true)
19120 .await
19121 .unwrap()
19122 .expect("the role rewrites after the implementer fails");
19123 assert!(msg.body.contains("`src/net.rs`"));
19124 assert_eq!(
19125 runner.state.seats["impl-A"].turns, 2,
19126 "the implementer was tried once"
19127 );
19128 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19129 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19130 }
19131
19132 #[tokio::test]
19133 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
19134 crate::run::pin_test_home();
19135 let dir = tempfile::tempdir().unwrap();
19136 let mut broken = spec("bad");
19137 broken.command = vec!["false".into()];
19138 let mut runner = rewrite_runner(
19139 dir.path(),
19140 true,
19141 vec![
19142 broken,
19143 scripted("noformat", "Add retries."),
19144 scripted("good", EN_REWRITE),
19145 ],
19146 );
19147 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19148 "bad".into(),
19149 "noformat".into(),
19150 "good".into(),
19151 ]));
19152 let winner = runner.state.candidates[0].clone();
19153 let msg = runner
19154 .guarded_pr_message(&winner, None, true)
19155 .await
19156 .unwrap()
19157 .expect("the third agent rewrites");
19158 assert!(msg.body.contains("`src/net.rs`"));
19159 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
19160 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
19161 }
19162
19163 #[tokio::test]
19164 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
19165 crate::run::pin_test_home();
19166 for (agents, role, saved) in [
19168 (
19169 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
19170 Some("beta"),
19171 true,
19172 ),
19173 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
19174 (Vec::new(), None, false),
19175 ] {
19176 let dir = tempfile::tempdir().unwrap();
19177 let mut runner = rewrite_runner(dir.path(), true, agents);
19178 runner.state.config.roles.language_judge =
19179 role.map(|r| crate::config::AgentChoice::One(r.into()));
19180 let winner = runner.state.candidates[0].clone();
19181 let first = runner
19182 .guarded_pr_message(&winner, None, true)
19183 .await
19184 .unwrap();
19185 assert!(first.is_none(), "withheld text parks the run");
19186 assert!(runner.state.parked);
19187 assert_eq!(gate_questions(&runner), 1);
19188 let g = runner.state.github_text.as_ref().unwrap();
19189 assert_eq!(g.rewrite.is_some(), saved);
19190 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
19191 }
19192 }
19193
19194 #[tokio::test]
19195 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
19196 crate::run::pin_test_home();
19197 let dir = tempfile::tempdir().unwrap();
19198 let mut runner = runner_at(RunStatus::Gating);
19199 runner.state =
19200 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
19201 runner.state.repo = dir.path().to_owned();
19202 runner.state.candidates[0].worktree = dir.path().to_owned();
19203 let mut author = spec("alpha");
19204 author.command = vec![
19205 "sh".into(),
19206 "-c".into(),
19207 "printf '%s' \"$REWRITE_REPLY\"".into(),
19208 ];
19209 author.env.insert(
19210 "REWRITE_REPLY".into(),
19211 format!(
19212 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19213 "English sentence. ".repeat(600)
19214 ),
19215 );
19216 runner.state.config.agents = vec![author];
19217 seed_public_summary(&mut runner);
19218 let winner = runner.state.candidates[0].clone();
19219 assert!(
19220 runner
19221 .guarded_pr_message(&winner, None, true)
19222 .await
19223 .unwrap()
19224 .is_none()
19225 );
19226 let record = runner.state.github_text.clone().unwrap();
19227 let rewrite = record.rewrite.clone().unwrap();
19228 assert_eq!(rewrite.failed, ["body-language"]);
19229 assert!(
19230 rewrite
19231 .body
19232 .ends_with(&format!("magi:run/{}", runner.state.id))
19233 );
19234 let store = ask::Questions::open();
19235 let q = store.get(record.question.as_ref().unwrap()).unwrap();
19236 assert!(
19237 q.choices
19238 .contains(&crate::github_text::USE_REWRITE.to_owned())
19239 );
19240 for label in [
19241 "ORIGINAL",
19242 "FALLBACK",
19243 "REWRITE",
19244 "Rules still failed: body-language",
19245 "truncated",
19246 ] {
19247 assert!(q.detail.contains(label), "{label}");
19248 }
19249 let artifact = std::fs::read_to_string(crate::run::artifact_path(
19250 &runner.state,
19251 &crate::github_text::artifact_name(&record.fingerprint),
19252 ))
19253 .unwrap();
19254 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19255 runner.state =
19257 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19258 store
19259 .update(&q.id, |q| {
19260 q.say("まだ日本語")?;
19261 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19262 })
19263 .unwrap();
19264 assert!(
19265 runner
19266 .guarded_pr_message(&winner, None, true)
19267 .await
19268 .unwrap()
19269 .is_none()
19270 );
19271 let retry = runner.state.github_text.as_ref().unwrap();
19272 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19273 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19274 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19275 assert!(
19276 q.posting_comparison
19277 .as_ref()
19278 .unwrap()
19279 .matches_detail(&q.detail)
19280 );
19281 store
19282 .update(&q.id, |q| {
19283 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19284 })
19285 .unwrap();
19286 let message = runner
19287 .guarded_pr_message(&winner, None, true)
19288 .await
19289 .unwrap()
19290 .unwrap();
19291 assert_eq!(message.title, rewrite.title);
19292 assert_eq!(message.body, rewrite.body);
19293 assert_eq!(
19294 runner.state.github_text.as_ref().unwrap().choice,
19295 Some(GithubTextChoice::Rewrite)
19296 );
19297 let again = runner
19298 .guarded_pr_message(&winner, None, true)
19299 .await
19300 .unwrap()
19301 .unwrap();
19302 assert_eq!(again.title, message.title);
19303 assert_eq!(again.body, message.body);
19304 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19305 assert!(
19306 !runner
19307 .state
19308 .events
19309 .iter()
19310 .any(|e| e.message.contains(&rewrite.title)
19311 || e.message.contains("English sentence."))
19312 );
19313 }
19314
19315 #[tokio::test]
19316 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19317 crate::run::pin_test_home();
19318 let dir = tempfile::tempdir().unwrap();
19319 let mut runner = runner_at(RunStatus::Gating);
19320 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19321 runner.state.repo = dir.path().to_owned();
19322 runner.state.candidates[0].worktree = dir.path().to_owned();
19323 seed_public_summary(&mut runner);
19324 let winner = runner.state.candidates[0].clone();
19325 assert!(
19326 runner
19327 .guarded_pr_message(&winner, None, true)
19328 .await
19329 .unwrap()
19330 .is_none()
19331 );
19332 let record = runner.state.github_text.as_ref().unwrap();
19333 assert!(record.rewrite.is_none());
19334 let mut old = serde_json::to_value(record).unwrap();
19336 old.as_object_mut().unwrap().remove("rewrite");
19337 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19338 assert!(old.rewrite.is_none());
19339 let id = record.question.clone().unwrap();
19340 ask::Questions::open()
19341 .update(&id, |q| {
19342 q.choices.push(crate::github_text::USE_REWRITE.into());
19343 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19344 })
19345 .unwrap();
19346 let message = runner
19347 .guarded_pr_message(&winner, None, true)
19348 .await
19349 .unwrap()
19350 .unwrap();
19351 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19352 assert_eq!(
19353 runner.state.github_text.as_ref().unwrap().choice,
19354 Some(GithubTextChoice::Fallback)
19355 );
19356 }
19357
19358 #[tokio::test]
19359 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19360 crate::run::pin_test_home();
19361 let dir = tempfile::tempdir().unwrap();
19362 let mut runner = runner_at(RunStatus::Gating);
19363 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19364 runner.state.repo = dir.path().to_owned();
19365 runner.state.candidates[0].worktree = dir.path().to_owned();
19366 seed_public_summary(&mut runner);
19367 let winner = runner.state.candidates[0].clone();
19368 assert!(
19369 runner
19370 .guarded_pr_message(&winner, None, true)
19371 .await
19372 .unwrap()
19373 .is_none()
19374 );
19375 let store = ask::Questions::open();
19376 let find = |run: &str| {
19377 store
19378 .list()
19379 .into_iter()
19380 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19381 .unwrap()
19382 };
19383 let run = runner.state.id.clone();
19384 let q = find(&run);
19385 let fp = runner
19386 .state
19387 .github_text
19388 .as_ref()
19389 .unwrap()
19390 .fingerprint
19391 .clone();
19392 let name = crate::github_text::artifact_name(&fp);
19393 let path = crate::run::artifact_path(&runner.state, &name);
19394 assert!(q.detail.contains(&path.display().to_string()));
19395 assert!(q.detail.contains("title-language: title"));
19396 let saved = std::fs::read_to_string(&path).expect("artifact written");
19397 assert!(saved.contains("日本語のタイトル"));
19398 assert!(q.cwd.is_none());
19399 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19400 let say = |id: &str, text: &str, choice: &str| {
19402 store
19403 .update(id, |q| {
19404 q.thread.push(ask::Turn {
19405 who: ask::Who::Operator,
19406 body: text.to_owned(),
19407 at: jiff::Timestamp::now(),
19408 note: None,
19409 });
19410 q.answer(ask::Answer::Choice(choice.to_owned()))
19411 })
19412 .unwrap();
19413 };
19414 say(&q.id, "まだ日本語", "use my text");
19415 assert!(
19416 runner
19417 .guarded_pr_message(&winner, None, true)
19418 .await
19419 .unwrap()
19420 .is_none()
19421 );
19422 let second = find(&run);
19423 assert_ne!(second.id, q.id);
19424 assert!(second.detail.contains("did not pass"));
19425 assert!(second.detail.contains("日本語のタイトル"));
19428 assert!(!second.detail.contains("まだ日本語"));
19429 assert!(second.detail.contains(&path.display().to_string()));
19430 say(&second.id, "fix: retry failed requests", "use my text");
19431 let message = runner
19432 .guarded_pr_message(&winner, None, true)
19433 .await
19434 .unwrap()
19435 .expect("a vetted replacement is posted");
19436 assert_eq!(message.title, "fix: retry failed requests");
19437 assert_eq!(
19438 runner
19439 .state
19440 .github_text
19441 .as_ref()
19442 .unwrap()
19443 .chosen_title
19444 .as_deref(),
19445 Some("fix: retry failed requests")
19446 );
19447 assert!(runner.state.github_text.as_ref().unwrap().resolved);
19448 let again = runner
19450 .guarded_pr_message(&winner, None, true)
19451 .await
19452 .unwrap()
19453 .unwrap();
19454 assert_eq!(again.title, message.title);
19455 assert_eq!(
19456 store
19457 .list()
19458 .into_iter()
19459 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19460 .count(),
19461 2
19462 );
19463 }
19464
19465 #[test]
19466 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19467 let mut state = state_with_summary(
19471 "add retries",
19472 "TITLE: fix(web): batch reads\n- reads run.json once",
19473 );
19474 state.config.graph.language = "ja".to_owned();
19475 let m = pr_message(&state, 'A');
19476 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19477 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19478
19479 let task = "今回やってほしいこと: results projector を直す";
19482 let mut state = state_with_summary(task, "- no title line");
19483 state.config.graph.language = "ja".to_owned();
19484 let m = pr_message(&state, 'A');
19485 assert_eq!(
19486 m.title,
19487 format!("chore: land candidate A of run {}", state.id)
19488 );
19489 assert!(
19490 m.body.contains(&format!(
19491 "<summary>Original task</summary>\n\n{task}\n\n</details>"
19492 )),
19493 "{}",
19494 m.body
19495 );
19496 }
19497
19498 #[test]
19499 fn pr_message_scrubs_home_paths_and_addresses() {
19500 let state = state_with_summary(
19501 "fix it in /Users/someone/src/x",
19502 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19503 );
19504 let m = pr_message(&state, 'A');
19505 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19506 assert!(!m.body.contains(leak), "{}", m.body);
19507 }
19508 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19509 }
19510
19511 #[test]
19512 fn pr_message_survives_a_task_that_closes_details() {
19513 let state = state_with_summary("a </details> b", "TITLE: fix: x");
19514 let m = pr_message(&state, 'A');
19515 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19516 }
19517
19518 #[test]
19519 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19520 let cmd = manual_merge_command(
19521 MergeStyle::Squash,
19522 Path::new("/repo"),
19523 "b",
19524 "fix: \"quoted\" $(x) `y`\n\nbody",
19525 );
19526 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19527 }
19528
19529 #[test]
19530 fn manual_merge_command_matches_the_configured_style() {
19531 let repo = Path::new("/repo");
19532 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19533
19534 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19535 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19536
19537 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19538 assert_eq!(
19539 squash,
19540 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19541 \"Merge magi run 0832 (candidate A)\""
19542 );
19543
19544 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19545 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19546 }
19547
19548 #[test]
19549 fn a_nudge_gets_a_quarter_of_the_budget() {
19550 assert_eq!(retry_budget(secs(1200), true), secs(300));
19552 assert_eq!(retry_budget(secs(3600), true), secs(900));
19553 }
19554
19555 #[test]
19556 fn a_resent_prompt_keeps_the_whole_budget() {
19557 assert_eq!(retry_budget(secs(1200), false), secs(1200));
19560 assert_eq!(retry_budget(secs(60), false), secs(60));
19561 }
19562
19563 #[test]
19564 fn the_floor_never_exceeds_the_original_budget() {
19565 assert_eq!(retry_budget(secs(60), true), secs(60));
19569 assert_eq!(retry_budget(secs(480), true), secs(120));
19570 assert_eq!(retry_budget(secs(0), true), secs(0));
19571 }
19572
19573 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19574 agent::CommandEvidence {
19575 id: "item1".to_owned(),
19576 description: "cargo test".to_owned(),
19577 exit_code,
19578 result_summary: String::new(),
19579 source: "codex".to_owned(),
19580 }
19581 }
19582
19583 #[test]
19584 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19585 assert!(!has_unconfirmed_command(&[]));
19589 }
19590
19591 #[test]
19592 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19593 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19597 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19598 assert!(!has_unconfirmed_command(&[
19599 evidence(Some(0)),
19600 evidence(Some(101))
19601 ]));
19602 }
19603
19604 #[test]
19605 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19606 assert!(has_unconfirmed_command(&[
19607 evidence(Some(0)),
19608 evidence(None)
19609 ]));
19610 }
19611
19612 #[test]
19613 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19614 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19615 assert_eq!(
19616 verified_noop_claim(true, &[], text).as_deref(),
19617 Some("already fixed by b32cfc4, on main.")
19618 );
19619 }
19620
19621 #[test]
19622 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19623 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19626 assert!(verified_noop_claim(false, &[], text).is_none());
19627 }
19628
19629 #[test]
19630 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19631 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19632 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19633 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19635 }
19636
19637 #[test]
19638 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19639 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19640 }
19641
19642 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19645 runner.state.candidates = shape
19646 .iter()
19647 .enumerate()
19648 .map(|(i, &(empty, verified))| Candidate {
19649 index: i,
19650 label: (b'A' + i as u8) as char,
19651 agent: "sonnet".to_owned(),
19652 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19653 worktree: PathBuf::from(format!("/wt/{i}")),
19654 summary: String::new(),
19655 stat: String::new(),
19656 files: 0,
19657 commits: 0,
19658 empty,
19659 failed: None,
19660 verified_noop: verified.map(str::to_owned),
19661 duration_ms: 0,
19662 folded: false,
19663 })
19664 .collect();
19665 }
19666
19667 #[test]
19668 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19669 ask_test_home();
19670 let mut runner = runner_at(RunStatus::Implementing);
19671 set_candidates(
19672 &mut runner,
19673 &[
19674 (true, Some("already on main at b32cfc4")),
19675 (true, Some("same fix, see the existing test")),
19676 ],
19677 );
19678
19679 runner
19680 .after_implement()
19681 .expect("a verified no-op is not an error");
19682
19683 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19684 }
19685
19686 #[test]
19687 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19688 ask_test_home();
19689 let mut runner = runner_at(RunStatus::Implementing);
19690 set_candidates(
19694 &mut runner,
19695 &[(true, Some("already on main at b32cfc4")), (true, None)],
19696 );
19697
19698 let err = runner
19699 .after_implement()
19700 .expect_err("an unverified empty candidate must still fail the run");
19701
19702 assert!(
19703 err.to_string().contains("no candidate produced a change"),
19704 "{err}"
19705 );
19706 assert_eq!(runner.state.status, RunStatus::Failed);
19707 }
19708
19709 #[test]
19710 fn after_implement_still_fails_an_ordinary_all_empty_run() {
19711 ask_test_home();
19712 let mut runner = runner_at(RunStatus::Implementing);
19713 set_candidates(&mut runner, &[(true, None), (true, None)]);
19714
19715 let err = runner
19716 .after_implement()
19717 .expect_err("no candidate declared anything; this is an ordinary failure");
19718
19719 assert!(
19720 err.to_string().contains("no candidate produced a change"),
19721 "{err}"
19722 );
19723 assert_eq!(runner.state.status, RunStatus::Failed);
19724 }
19725}
19726
19727#[derive(Debug)]
19730struct GraftRestoreFailed(String);
19731
19732impl std::fmt::Display for GraftRestoreFailed {
19733 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19734 write!(
19735 f,
19736 "could not restore the winner to its pre-graft commit: {}",
19737 self.0
19738 )
19739 }
19740}
19741
19742impl std::error::Error for GraftRestoreFailed {}
19743
19744#[cfg(test)]
19748mod resource_wait_tests {
19749 use super::*;
19750 use std::collections::BTreeMap;
19751
19752 fn sh(id: &str, script: &str) -> AgentSpec {
19753 AgentSpec {
19754 id: id.to_owned(),
19755 kind: crate::config::AgentKind::Command,
19756 model: None,
19757 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19758 extra_args: Vec::new(),
19759 env: BTreeMap::new(),
19760 prompt_delivery: None,
19761 max_turn_secs: None,
19762 }
19763 }
19764
19765 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19766 SeatJob {
19767 seat: SeatState::new("impl-A", &spec.id, 1),
19768 spec,
19769 cwd: dir.to_path_buf(),
19770 prompt: "do it".to_owned(),
19771 timeout,
19772 allow_write: true,
19773 sessions: false,
19774 artifacts: dir.join("art"),
19775 stem: "impl-A".to_owned(),
19776 handover: None,
19777 }
19778 }
19779
19780 fn state(resource_wait_secs: u64) -> RunState {
19781 let mut config = Config::default();
19782 config.graph.timeout_resource_wait = resource_wait_secs;
19783 RunState::new(
19784 PathBuf::from("/repo"),
19785 "main".to_owned(),
19786 "abc1234".to_owned(),
19787 "task".to_owned(),
19788 config,
19789 )
19790 }
19791
19792 fn hold(cache: &Path) -> crate::cache::Guard {
19793 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19794 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19795 crate::cache::AcquireOutcome::Acquired(g) => g,
19796 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19797 }
19798 }
19799
19800 async fn run(
19801 jobs: Vec<SeatJob>,
19802 sem: Arc<Semaphore>,
19803 cache: Option<&Path>,
19804 st: &mut RunState,
19805 ) -> Vec<(usize, SeatState, AgentOutcome)> {
19806 let prompts = Prompts::default();
19807 let ctx = WaveCtx {
19808 run: "r",
19809 node: "implement",
19810 prompts: &prompts,
19811 cache,
19812 round: None,
19813 carry_seats: false,
19814 pauses: None,
19815 };
19816 wave(jobs, sem, &ctx, st, 0).await
19817 }
19818
19819 #[tokio::test]
19820 async fn verification_cache_wait_preserves_the_command_budget() {
19821 crate::run::pin_test_home();
19822 let tmp = tempfile::tempdir().unwrap();
19823 let cache = tmp.path().join("target");
19824 let guard = hold(&cache);
19825 let release = tokio::spawn(async move {
19826 tokio::time::sleep(Duration::from_millis(100)).await;
19827 guard.release();
19828 });
19829 let mut st = state(10);
19830 let budget = Duration::from_millis(50);
19831 let wait = repository_wait(&st, None);
19832 let (outcomes, _) = with_cache_lease(
19833 &mut st,
19834 Some(&cache),
19835 "gate",
19836 "gate",
19837 tmp.path(),
19838 "abc1234",
19839 budget,
19840 "test gate",
19841 &wait,
19842 |_, available| async move {
19843 assert_eq!(
19844 available, budget,
19845 "contention must not shorten verification"
19846 );
19847 (Vec::new(), false, Vec::new())
19848 },
19849 )
19850 .await
19851 .unwrap();
19852 release.await.unwrap();
19853 assert!(outcomes.is_empty(), "{outcomes:?}");
19854 assert!(
19855 st.events
19856 .iter()
19857 .any(|event| event.message.contains("waiting for"))
19858 );
19859 assert!(st.resource_wait.is_none());
19860 hold(&cache).release();
19862 }
19863
19864 #[tokio::test]
19865 async fn verification_cache_wait_expires_without_starting_commands() {
19866 crate::run::pin_test_home();
19867 let tmp = tempfile::tempdir().unwrap();
19868 let cache = tmp.path().join("target");
19869 let _held = hold(&cache);
19870 let mut st = state(0);
19871 let wait = repository_wait(&st, None);
19872 let (outcomes, _) = with_cache_lease(
19873 &mut st,
19874 Some(&cache),
19875 "gate",
19876 "gate",
19877 tmp.path(),
19878 "abc1234",
19879 Duration::from_secs(60),
19880 "test gate",
19881 &wait,
19882 |_, _| async { panic!("commands must not start without the cache lease") },
19883 )
19884 .await
19885 .unwrap();
19886 assert_eq!(outcomes.len(), 1);
19887 assert!(outcomes[0].resource_blocked);
19888 assert!(st.resource_wait.is_none());
19889 assert!(matches!(
19890 st.resource_block.unwrap().cause,
19891 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19892 ));
19893 }
19894
19895 #[tokio::test]
19896 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19897 crate::run::pin_test_home();
19898 let tmp = tempfile::tempdir().unwrap();
19899 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19900 let cache = tmp.path().join("cache");
19901 let mut state = state(21600);
19902 let wait = repository_wait(&state, None);
19903 let (outcomes, _) = with_cache_lease(
19904 &mut state,
19905 Some(&cache),
19906 "gate",
19907 "gate",
19908 tmp.path(),
19909 "head",
19910 Duration::from_secs(60),
19911 "gate",
19912 &wait,
19913 |_, _| async { panic!("freshness failure must not start verification") },
19914 )
19915 .await
19916 .unwrap();
19917 assert_eq!(outcomes.len(), 1);
19918 assert!(outcomes[0].resource_blocked);
19919 let block = state.resource_block.unwrap();
19920 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19921 assert!(!block.description().contains("timed out"));
19922 }
19923
19924 #[tokio::test]
19925 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19926 crate::run::pin_test_home();
19927 let tmp = tempfile::tempdir().unwrap();
19928 let cache = tmp.path().join("cache");
19929 let _held = hold(&cache);
19930 let mut state = state(21600);
19931 let pause = Pause::new();
19932 let interrupt = Pause::new();
19933 let cause = ParkCause::Operator;
19934 pause.park_for(cause.clone());
19935 let prompts = Prompts::default();
19936 let ctx = WaveCtx {
19937 run: "test",
19938 node: "fix",
19939 prompts: &prompts,
19940 cache: Some(&cache),
19941 round: None,
19942 carry_seats: false,
19943 pauses: Some((&pause, &interrupt)),
19944 };
19945 let result = wave(
19946 vec![job(
19947 sh("writer", "exit 99"),
19948 tmp.path(),
19949 Duration::from_secs(1),
19950 )],
19951 Arc::new(Semaphore::new(1)),
19952 &ctx,
19953 &mut state,
19954 0,
19955 )
19956 .await;
19957 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19958 assert_eq!(result[0].1.turns, 0);
19959 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19960 assert_eq!(
19961 state.resource_block.unwrap().cause,
19962 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19963 );
19964 assert!(state.active.is_empty());
19965 }
19966
19967 #[tokio::test]
19968 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19969 crate::run::pin_test_home();
19970 let tmp = tempfile::tempdir().unwrap();
19971 let cache = tmp.path().join("target-a");
19972 let guard = hold(&cache);
19973 tokio::spawn(async move {
19975 tokio::time::sleep(Duration::from_millis(1500)).await;
19976 guard.release();
19977 });
19978 let mut st = state(60);
19979 let results = run(
19980 vec![job(
19981 sh("alpha", "echo ok"),
19982 tmp.path(),
19983 Duration::from_secs(1),
19984 )],
19985 Arc::new(Semaphore::new(1)),
19986 Some(&cache),
19987 &mut st,
19988 )
19989 .await;
19990 assert!(
19991 matches!(results[0].2, AgentOutcome::Ok(_)),
19992 "the wait must not be charged to the agent"
19993 );
19994 assert!(st.resource_block.is_none());
19995 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19996 assert!(st.resource_wait.is_none() && st.active.is_empty());
19997 }
19998
19999 #[tokio::test]
20000 #[cfg(unix)]
20001 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20002 use std::os::fd::AsRawFd as _;
20003 crate::run::pin_test_home();
20004 let tmp = tempfile::tempdir().unwrap();
20005 crate::git::git(tmp.path(), &["init"]).await.unwrap();
20006 let common = tmp.path().join(".git");
20007 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20008 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20009 assert_eq!(
20010 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20011 0
20012 );
20013 let mut st = state(1);
20014 st.config.coordination = crate::config::test_coordination();
20015 let mut reader = job(
20016 sh("reader", "echo must-not-run"),
20017 tmp.path(),
20018 Duration::from_secs(30),
20019 );
20020 reader.allow_write = false;
20021 let results = run(
20022 vec![reader.clone()],
20023 Arc::new(Semaphore::new(1)),
20024 None,
20025 &mut st,
20026 )
20027 .await;
20028 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20029 assert_eq!(results[0].1.turns, 0);
20030 assert!(FailClass::of(&results[0].2).is_none());
20031 assert_eq!(
20032 st.resource_block.as_ref().unwrap().kind,
20033 "repository coordination"
20034 );
20035 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20036 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20037 drop(external);
20038 let editor = crate::repo_lock::SeatLease::for_checkout(
20039 tmp.path(),
20040 true,
20041 crate::config::test_coordination().as_ref(),
20042 )
20043 .await
20044 .unwrap();
20045 let pause = Pause::new();
20046 let interrupt = Pause::new();
20047 let prompts = Prompts::default();
20048 let ctx = WaveCtx {
20049 run: "r",
20050 node: "review",
20051 prompts: &prompts,
20052 cache: None,
20053 round: None,
20054 carry_seats: false,
20055 pauses: Some((&pause, &interrupt)),
20056 };
20057 let (results, ()) = tokio::join!(
20058 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20059 async {
20060 tokio::time::sleep(Duration::from_millis(50)).await;
20061 pause.park();
20062 }
20063 );
20064 assert!(
20065 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20066 );
20067 assert_eq!(results[0].1.turns, 0);
20068 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20069 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20070 drop(editor);
20071 }
20072
20073 #[tokio::test]
20074 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20075 crate::run::pin_test_home();
20076 let tmp = tempfile::tempdir().unwrap();
20077 let cache = tmp.path().join("target-b");
20078 let _held = hold(&cache);
20079 let mut st = state(1);
20080 let results = run(
20081 vec![job(
20082 sh("alpha", "echo ok"),
20083 tmp.path(),
20084 Duration::from_secs(1),
20085 )],
20086 Arc::new(Semaphore::new(1)),
20087 Some(&cache),
20088 &mut st,
20089 )
20090 .await;
20091 let (_, seat, out) = &results[0];
20092 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
20093 assert!(wave_blocked(&results));
20094 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
20095 assert_eq!(seat.turns, 0, "nothing was asked");
20096 assert!(
20097 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
20098 "no agent was spawned"
20099 );
20100 assert!(FailClass::of(out).is_none(), "not a failure class");
20101 let block = st.resource_block.as_ref().expect("recorded for the resume");
20102 assert_eq!(block.kind, "shared build cache");
20103 assert!(matches!(
20104 block.cause,
20105 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
20106 ));
20107 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20108 assert!(st.active.is_empty());
20109 }
20110
20111 #[tokio::test]
20112 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
20113 crate::run::pin_test_home();
20114 let tmp = tempfile::tempdir().unwrap();
20115 let mut st = state(1);
20116 let results = run(
20117 vec![job(
20118 sh("alpha", "echo ok"),
20119 tmp.path(),
20120 Duration::from_secs(1),
20121 )],
20122 Arc::new(Semaphore::new(0)),
20123 None,
20124 &mut st,
20125 )
20126 .await;
20127 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20128 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
20129 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
20130 }
20131
20132 #[tokio::test]
20133 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
20134 crate::run::pin_test_home();
20135 let tmp = tempfile::tempdir().unwrap();
20136 let mut st = state(60);
20137 let results = run(
20138 vec![job(
20139 sh("alpha", "sleep 30"),
20140 tmp.path(),
20141 Duration::from_secs(1),
20142 )],
20143 Arc::new(Semaphore::new(1)),
20144 None,
20145 &mut st,
20146 )
20147 .await;
20148 let out = &results[0].2;
20149 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
20150 let class = FailClass::of(out).expect("a failure");
20151 assert_eq!(class, FailClass::Timeout);
20152 assert!(should_hand_over(None, &class), "existing handover policy");
20153 assert!(st.resource_block.is_none());
20154 }
20155}